In [ ]:
#PROBLEM STATEMENT : The main aim is to detect the aeroplane in the image.
In [1]:
# to select the version of tensor flow
%tensorflow_version 1.x
TensorFlow 1.x selected.
In [2]:
# Some models to train on
MODELS_CONFIG = {
'ssd_mobilenet_v2': {
'model_name': 'ssd_mobilenet_v2_coco_2018_03_29', # Layers input and output shape how many neurons in each layer(Model building)
'pipeline_file': 'ssd_mobilenet_v2_coco.config', # While compiling and fitting the model some parameters should be tuned (Defining the arguments values)
},
'faster_rcnn_inception_v2': {
'model_name': 'faster_rcnn_inception_v2_coco_2018_01_28',
'pipeline_file': 'faster_rcnn_inception_v2_pets.config',
},
'rfcn_resnet101': {
'model_name': 'rfcn_resnet101_coco_2018_01_28',
'pipeline_file': 'rfcn_resnet101_pets.config',
}
}
# Select a model in `MODELS_CONFIG`.
# I chose ssd_mobilenet_v2 for this project, you could choose any
selected_model = 'ssd_mobilenet_v2'
In [3]:
!apt-get install -qq protobuf-compiler python-pil python-lxml python-tk
!pip install -qq Cython contextlib2 pillow lxml matplotlib
!pip install -qq pycocotools
E: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/p/pillow/python-pil_5.1.0-1ubuntu0.2_amd64.deb 404 Not Found [IP: 91.189.88.142 80] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
In [4]:
# importing the packages
from __future__ import division, print_function, absolute_import
import pandas as pd
import numpy as np
import csv
import re
import cv2
import os
import glob
import xml.etree.ElementTree as ET
import io
import tensorflow.compat.v1 as tf
from PIL import Image
from collections import namedtuple, OrderedDict
import shutil
import urllib.request
import tarfile
from google.colab import files
In [5]:
#we need tenorflow v 1.15.0, object detection API is removed from tf v 2.0+
#!pip install tensorflow==1.15.0
print(tf.__version__)
1.15.2
In [6]:
#creates a directory for the whole project
!mkdir aeroplane_detection
In [7]:
# list information about files and directories within the file system.
!ls -l
total 12 drwxr-xr-x 2 root root 4096 Jul 25 06:21 aeroplane_detection drwx------ 4 root root 4096 Jul 25 06:19 drive drwxr-xr-x 1 root root 4096 Jul 10 16:29 sample_data
In [8]:
# changing the current directory
%cd /content/aeroplane_detection
/content/aeroplane_detection
In [9]:
# creating a directory to store the training and testing data
!mkdir data
# folders for the training and testing data.
!mkdir data/images data/train_labels data/test_labels
In [10]:
# list information about files and directories within the file system.
!ls
data
In [11]:
# present working directly
!pwd
/content/aeroplane_detection
In [12]:
# importing the os to check the files in the directory
import os
os.listdir('data')
Out[12]:
['train_labels', 'images', 'test_labels']
In [13]:
# extracting the the images zip files from the drive and storing it in the tmp folder
import os
import zipfile
local_zip = "/content/drive/My Drive/Images.zip" ## which zip file you want to extarct
zip_ref = zipfile.ZipFile(local_zip,'r')
zip_ref.extractall('/tmp') ## In which location you want to store extracted files
zip_ref.close()
In [14]:
# extracting the the Airplanes_Annotations zip files from the drive and storing it in the tmp folder
import os
import zipfile
local_zip = "/content/drive/My Drive/Airplane_Annotation.zip" ## which zip file you want to extarct
zip_ref = zipfile.ZipFile(local_zip,'r')
zip_ref.extractall('/tmp') ## In which location you want to store extracted files
zip_ref.close()
In [15]:
# checking the images folder whether present or not
os.listdir("/tmp/Images")
Out[15]:
['airplane_371.jpg', 'airplane_357.jpg', 'airplane_116.jpg', 'airplane_507.jpg', 'airplane_034.jpg', 'airplane_675.jpg', 'airplane_483.jpg', 'airplane_642.jpg', 'airplane_031.jpg', 'airplane_324.jpg', 'airplane_303.jpg', 'airplane_370.jpg', 'airplane_647.jpg', 'airplane_032.jpg', 'airplane_133.jpg', 'airplane_232.jpg', 'airplane_564.jpg', 'airplane_185.jpg', 'Planes12.jpg', 'airplane_058.jpg', 'airplane_108.jpg', 'airplane_510.jpg', 'airplane_354.jpg', 'airplane_674.jpg', 'airplane_187.jpg', 'Planes6.jpg', 'airplane_288.jpg', 'airplane_410.jpg', 'airplane_162.jpg', 'airplane_122.jpg', 'airplane_514.jpg', 'airplane_171.jpg', 'airplane_629.jpg', 'airplane_035.jpg', 'airplane_579.jpg', 'airplane_687.jpg', 'airplane_545.jpg', 'airplane_033.jpg', 'airplane_553.jpg', 'airplane_574.jpg', 'airplane_019.jpg', 'airplane_125.jpg', 'airplane_252.jpg', 'airplane_513.jpg', 'airplane_316.jpg', 'airplane_161.jpg', 'airplane_677.jpg', 'airplane_087.jpg', 'Planes3.jpg', 'airplane_620.jpg', 'airplane_462.jpg', 'airplane_550.jpg', 'airplane_549.jpg', 'airplane_179.jpg', 'airplane_338.jpg', 'airplane_305.jpg', 'airplane_661.jpg', 'airplane_653.jpg', 'airplane_336.jpg', 'airplane_245.jpg', 'airplane_368.jpg', 'airplane_199.jpg', 'airplane_608.jpg', 'airplane_694.jpg', 'airplane_092.jpg', 'airplane_560.jpg', 'airplane_700.jpg', 'airplane_443.jpg', 'airplane_211.jpg', 'airplane_465.jpg', 'airplane_696.jpg', 'airplane_334.jpg', 'airplane_688.jpg', 'airplane_597.jpg', 'airplane_555.jpg', 'airplane_281.jpg', 'airplane_066.jpg', 'airplane_276.jpg', 'airplane_165.jpg', 'airplane_421.jpg', 'airplane_422.jpg', 'airplane_271.jpg', 'airplane_673.jpg', 'airplane_404.jpg', 'airplane_460.jpg', 'airplane_206.jpg', 'airplane_419.jpg', 'airplane_627.jpg', 'airplane_386.jpg', 'airplane_649.jpg', 'Planes7.jpg', 'airplane_542.jpg', 'airplane_475.jpg', 'airplane_282.jpg', 'airplane_083.jpg', 'airplane_331.jpg', 'airplane_690.jpg', 'airplane_416.jpg', 'airplane_028.jpg', 'airplane_384.jpg', 'airplane_626.jpg', 'airplane_503.jpg', 'airplane_473.jpg', 'airplane_415.jpg', 'airplane_449.jpg', 'airplane_256.jpg', 'airplane_246.jpg', 'airplane_401.jpg', 'airplane_652.jpg', 'airplane_628.jpg', 'airplane_509.jpg', 'airplane_594.jpg', 'airplane_445.jpg', 'airplane_296.jpg', 'airplane_102.jpg', 'Planes1.jpg', 'airplane_085.jpg', 'airplane_061.jpg', 'airplane_321.jpg', 'airplane_691.jpg', 'airplane_566.jpg', 'airplane_319.jpg', 'airplane_013.jpg', 'airplane_548.jpg', 'airplane_072.jpg', 'airplane_132.jpg', 'airplane_458.jpg', 'airplane_400.jpg', 'airplane_640.jpg', 'airplane_138.jpg', 'airplane_489.jpg', 'airplane_446.jpg', 'airplane_237.jpg', 'airplane_023.jpg', 'airplane_372.jpg', 'airplane_599.jpg', 'airplane_186.jpg', 'airplane_310.jpg', 'airplane_197.jpg', 'airplane_214.jpg', 'airplane_248.jpg', 'airplane_604.jpg', 'airplane_559.jpg', 'airplane_556.jpg', 'airplane_535.jpg', 'airplane_055.jpg', 'airplane_531.jpg', 'airplane_159.jpg', 'airplane_139.jpg', 'airplane_676.jpg', 'airplane_345.jpg', 'airplane_434.jpg', 'airplane_297.jpg', 'airplane_174.jpg', 'airplane_236.jpg', 'airplane_580.jpg', 'airplane_123.jpg', 'airplane_472.jpg', 'airplane_502.jpg', 'airplane_645.jpg', 'airplane_012.jpg', '428451.jpg', 'airplane_039.jpg', 'airplane_337.jpg', 'airplane_487.jpg', 'airplane_262.jpg', 'airplane_113.jpg', 'airplane_301.jpg', 'airplane_332.jpg', 'airplane_616.jpg', 'airplane_528.jpg', 'airplane_366.jpg', 'airplane_587.jpg', 'airplane_585.jpg', 'airplane_118.jpg', 'airplane_325.jpg', 'airplane_036.jpg', 'airplane_343.jpg', 'airplane_420.jpg', 'airplane_546.jpg', 'airplane_424.jpg', 'airplane_244.jpg', 'airplane_613.jpg', 'airplane_389.jpg', 'airplane_254.jpg', 'airplane_274.jpg', 'airplane_037.jpg', 'airplane_662.jpg', 'airplane_249.jpg', 'airplane_021.jpg', 'airplane_234.jpg', 'airplane_466.jpg', 'airplane_543.jpg', 'airplane_547.jpg', 'airplane_630.jpg', 'airplane_002.jpg', 'airplane_485.jpg', 'airplane_598.jpg', 'airplane_284.jpg', 'airplane_536.jpg', 'airplane_069.jpg', 'airplane_040.jpg', 'airplane_309.jpg', 'airplane_191.jpg', 'airplane_203.jpg', 'airplane_655.jpg', 'airplane_086.jpg', 'airplane_308.jpg', 'airplane_379.jpg', 'airplane_425.jpg', 'airplane_115.jpg', 'airplane_221.jpg', 'airplane_202.jpg', 'airplane_383.jpg', 'airplane_233.jpg', 'airplane_318.jpg', 'airplane_293.jpg', 'airplane_440.jpg', 'airplane_429.jpg', 'airplane_501.jpg', 'airplane_506.jpg', 'airplane_182.jpg', 'Planes11.jpg', 'airplane_226.jpg', 'airplane_508.jpg', '42848.jpg', 'airplane_062.jpg', 'airplane_495.jpg', 'airplane_409.jpg', 'airplane_243.jpg', 'airplane_657.jpg', 'airplane_617.jpg', 'airplane_381.jpg', 'airplane_098.jpg', 'airplane_263.jpg', 'airplane_082.jpg', 'airplane_065.jpg', 'airplane_431.jpg', 'airplane_093.jpg', 'airplane_198.jpg', 'airplane_114.jpg', 'airplane_444.jpg', 'airplane_486.jpg', 'airplane_364.jpg', 'airplane_079.jpg', 'airplane_363.jpg', 'airplane_157.jpg', 'airplane_533.jpg', 'airplane_572.jpg', 'airplane_643.jpg', 'airplane_349.jpg', 'airplane_377.jpg', 'airplane_052.jpg', 'airplane_047.jpg', 'airplane_024.jpg', 'Planes10.jpg', 'airplane_016.jpg', 'airplane_151.jpg', 'airplane_435.jpg', 'airplane_362.jpg', 'airplane_229.jpg', 'airplane_500.jpg', 'airplane_312.jpg', 'airplane_361.jpg', 'airplane_190.jpg', 'airplane_048.jpg', 'airplane_600.jpg', 'airplane_150.jpg', 'airplane_136.jpg', 'airplane_417.jpg', 'airplane_030.jpg', 'airplane_107.jpg', 'airplane_105.jpg', 'airplane_586.jpg', 'airplane_575.jpg', 'hotanairport5.jpg', 'airplane_350.jpg', 'airplane_317.jpg', 'airplane_180.jpg', 'airplane_682.jpg', 'airplane_447.jpg', 'airplane_530.jpg', 'airplane_212.jpg', 'airplane_470.jpg', 'airplane_686.jpg', 'airplane_049.jpg', 'airplane_265.jpg', 'airplane_140.jpg', 'airplane_639.jpg', 'airplane_053.jpg', 'airplane_432.jpg', 'airplane_591.jpg', 'airplane_664.jpg', 'airplane_552.jpg', 'airplane_450.jpg', 'airplane_605.jpg', 'airplane_418.jpg', 'airplane_026.jpg', 'airplane_679.jpg', 'airplane_601.jpg', 'hotanairport2-rgb.jpg', 'airplane_170.jpg', 'airplane_015.jpg', 'airplane_275.jpg', 'airplane_478.jpg', 'airplane_056.jpg', 'airplane_663.jpg', 'airplane_436.jpg', 'airplane_017.jpg', 'airplane_612.jpg', 'airplane_573.jpg', 'airplane_352.jpg', 'airplane_038.jpg', '42849.jpg', 'airplane_477.jpg', 'airplane_228.jpg', 'airplane_149.jpg', 'airplane_588.jpg', 'airplane_456.jpg', 'airplane_631.jpg', 'airplane_067.jpg', 'airplane_625.jpg', 'airplane_636.jpg', 'airplane_173.jpg', 'airplane_320.jpg', 'airplane_137.jpg', 'airplane_004.jpg', 'airplane_176.jpg', 'airplane_298.jpg', 'airplane_515.jpg', 'airplane_006.jpg', 'airplane_209.jpg', 'airplane_003.jpg', 'airplane_200.jpg', 'airplane_046.jpg', 'airplane_353.jpg', '428462.jpg', 'airplane_100.jpg', 'airplane_637.jpg', 'airplane_103.jpg', 'airplane_347.jpg', 'airplane_659.jpg', 'airplane_699.jpg', 'airplane_461.jpg', 'airplane_054.jpg', 'airplane_247.jpg', 'airplane_110.jpg', 'airplane_474.jpg', 'airplane_451.jpg', 'airplane_146.jpg', 'airplane_622.jpg', 'airplane_127.jpg', 'airplane_615.jpg', 'airplane_428.jpg', 'airplane_306.jpg', 'airplane_094.jpg', 'airplane_413.jpg', 'airplane_656.jpg', 'airplane_623.jpg', 'airplane_128.jpg', '428491.jpg', 'airplane_183.jpg', 'airplane_393.jpg', 'airplane_255.jpg', 'airplane_242.jpg', 'airplane_218.jpg', 'airplane_260.jpg', 'airplane_207.jpg', 'planes2.jpg', 'airplane_414.jpg', 'airplane_448.jpg', 'airplane_398.jpg', 'airplane_090.jpg', 'airplane_215.jpg', 'airplane_217.jpg', 'airplane_406.jpg', 'airplane_646.jpg', 'airplane_558.jpg', 'airplane_484.jpg', 'airplane_261.jpg', 'airplane_464.jpg', 'airplane_141.jpg', 'airplane_610.jpg', 'airplane_532.jpg', 'airplane_029.jpg', 'airplane_376.jpg', 'airplane_595.jpg', 'airplane_644.jpg', 'airplane_259.jpg', 'airplane_302.jpg', 'airplane_011.jpg', 'airplane_143.jpg', 'airplane_340.jpg', 'airplane_395.jpg', 'airplane_078.jpg', 'airplane_526.jpg', 'airplane_147.jpg', 'airplane_433.jpg', 'airplane_405.jpg', 'airplane_342.jpg', 'airplane_196.jpg', 'airplane_392.jpg', 'airplane_408.jpg', 'airplane_124.jpg', 'airplane_678.jpg', 'airplane_563.jpg', 'airplane_457.jpg', 'airplane_287.jpg', 'airplane_304.jpg', 'airplane_076.jpg', 'airplane_467.jpg', '428481.jpg', 'airplane_313.jpg', 'airplane_602.jpg', 'airplane_648.jpg', 'airplane_195.jpg', 'airplane_314.jpg', 'airplane_299.jpg', 'airplane_241.jpg', 'airplane_270.jpg', 'airplane_175.jpg', 'airplane_144.jpg', '428483.jpg', 'airplane_570.jpg', 'airplane_369.jpg', 'airplane_498.jpg', 'airplane_562.jpg', 'airplane_385.jpg', 'airplane_135.jpg', 'airplane_493.jpg', 'airplane_578.jpg', 'airplane_453.jpg', 'airplane_057.jpg', 'airplane_155.jpg', 'airplane_359.jpg', 'airplane_283.jpg', 'airplane_095.jpg', 'airplane_267.jpg', 'hotanairport1-rgb.jpg', 'airplane_224.jpg', 'airplane_311.jpg', 'airplane_084.jpg', 'airplane_292.jpg', 'airplane_059.jpg', 'hotanairport5-rgb.jpg', 'airplane_360.jpg', 'airplane_632.jpg', 'airplane_665.jpg', '428482.jpg', 'airplane_568.jpg', 'airplane_491.jpg', 'airplane_454.jpg', 'airplane_387.jpg', 'airplane_521.jpg', 'airplane_117.jpg', 'airplane_001.jpg', 'airplane_134.jpg', 'airplane_660.jpg', 'airplane_009.jpg', 'airplane_430.jpg', 'airplane_129.jpg', 'airplane_044.jpg', 'airplane_339.jpg', 'airplane_407.jpg', 'airplane_258.jpg', 'airplane_273.jpg', 'airplane_022.jpg', 'airplane_335.jpg', 'airplane_459.jpg', 'airplane_120.jpg', 'airplane_280.jpg', 'airplane_698.jpg', 'airplane_193.jpg', '42847.jpg', 'airplane_494.jpg', 'airplane_294.jpg', 'airplane_327.jpg', 'airplane_638.jpg', 'airplane_219.jpg', 'airplane_220.jpg', 'airplane_295.jpg', 'airplane_525.jpg', 'airplane_382.jpg', 'airplane_163.jpg', 'airplane_538.jpg', 'airplane_426.jpg', 'airplane_692.jpg', 'airplane_300.jpg', 'airplane_315.jpg', '428461.jpg', 'airplane_479.jpg', 'airplane_330.jpg', 'airplane_008.jpg', 'airplane_192.jpg', 'airplane_505.jpg', '428503.jpg', 'airplane_589.jpg', 'airplane_089.jpg', 'airplane_684.jpg', 'airplane_524.jpg', 'airplane_096.jpg', 'airplane_504.jpg', 'airplane_452.jpg', 'airplane_402.jpg', 'airplane_468.jpg', 'airplane_348.jpg', 'airplane_412.jpg', 'airplane_154.jpg', 'airplane_257.jpg', 'airplane_367.jpg', 'airplane_537.jpg', 'airplane_253.jpg', 'airplane_266.jpg', 'airplane_569.jpg', '42850.jpg', 'airplane_142.jpg', 'airplane_697.jpg', 'airplane_411.jpg', 'airplane_351.jpg', 'airplane_399.jpg', 'airplane_482.jpg', 'airplane_264.jpg', 'airplane_388.jpg', 'airplane_496.jpg', 'airplane_681.jpg', 'airplane_091.jpg', 'airplane_043.jpg', 'airplane_609.jpg', 'airplane_571.jpg', 'airplane_080.jpg', 'airplane_131.jpg', 'airplane_213.jpg', 'airplane_651.jpg', 'airplane_071.jpg', 'airplane_051.jpg', 'airplane_250.jpg', 'airplane_544.jpg', 'airplane_576.jpg', '428492.jpg', 'airplane_654.jpg', 'airplane_290.jpg', 'airplane_041.jpg', 'airplane_158.jpg', 'airplane_251.jpg', 'airplane_077.jpg', 'airplane_225.jpg', 'airplane_463.jpg', 'airplane_614.jpg', 'airplane_583.jpg', 'airplane_223.jpg', 'airplane_104.jpg', 'airplane_169.jpg', 'airplane_670.jpg', '428472.jpg', 'airplane_081.jpg', 'airplane_439.jpg', 'airplane_073.jpg', 'airplane_693.jpg', 'airplane_167.jpg', 'airplane_358.jpg', 'airplane_427.jpg', 'airplane_210.jpg', 'airplane_373.jpg', 'airplane_204.jpg', 'airplane_291.jpg', 'airplane_592.jpg', 'airplane_130.jpg', 'airplane_565.jpg', 'airplane_438.jpg', 'airplane_160.jpg', 'airplane_097.jpg', 'airplane_208.jpg', 'Planes9.jpg', 'airplane_050.jpg', 'airplane_166.jpg', 'airplane_106.jpg', 'airplane_455.jpg', 'airplane_240.jpg', 'airplane_075.jpg', 'airplane_333.jpg', 'airplane_391.jpg', 'airplane_177.jpg', 'airplane_356.jpg', 'airplane_201.jpg', 'Planes8.jpg', 'airplane_145.jpg', 'airplane_064.jpg', 'airplane_611.jpg', 'airplane_344.jpg', 'airplane_423.jpg', 'airplane_519.jpg', 'airplane_669.jpg', 'airplane_111.jpg', 'airplane_378.jpg', 'airplane_511.jpg', 'airplane_235.jpg', 'airplane_534.jpg', 'airplane_540.jpg', 'airplane_680.jpg', 'airplane_020.jpg', 'airplane_584.jpg', 'airplane_672.jpg', 'airplane_671.jpg', 'airplane_238.jpg', 'airplane_441.jpg', 'airplane_551.jpg', 'airplane_606.jpg', 'airplane_269.jpg', 'airplane_437.jpg', 'airplane_109.jpg', 'airplane_557.jpg', 'airplane_471.jpg', 'airplane_374.jpg', 'airplane_518.jpg', 'airplane_010.jpg', 'airplane_624.jpg', 'airplane_278.jpg', 'airplane_554.jpg', 'airplane_577.jpg', 'airplane_205.jpg', 'airplane_045.jpg', 'airplane_442.jpg', 'airplane_590.jpg', 'airplane_607.jpg', 'airplane_148.jpg', 'airplane_633.jpg', 'airplane_018.jpg', 'airplane_074.jpg', 'airplane_126.jpg', 'airplane_112.jpg', 'airplane_227.jpg', 'airplane_184.jpg', 'airplane_596.jpg', 'airplane_695.jpg', 'airplane_499.jpg', 'airplane_231.jpg', 'airplane_582.jpg', 'airplane_689.jpg', 'airplane_567.jpg', 'airplane_683.jpg', 'airplane_539.jpg', 'airplane_593.jpg', 'airplane_365.jpg', 'airplane_194.jpg', 'airplane_099.jpg', 'airplane_641.jpg', 'airplane_268.jpg', 'airplane_561.jpg', 'airplane_618.jpg', 'airplane_216.jpg', 'airplane_469.jpg', 'airplane_063.jpg', 'airplane_152.jpg', 'airplane_005.jpg', 'airplane_307.jpg', 'airplane_581.jpg', 'airplane_346.jpg', 'airplane_527.jpg', 'airplane_322.jpg', 'airplane_666.jpg', 'airplane_522.jpg', 'airplane_285.jpg', 'airplane_380.jpg', 'airplane_230.jpg', 'airplane_025.jpg', 'airplane_326.jpg', 'airplane_289.jpg', 'airplane_068.jpg', 'airplane_070.jpg', 'airplane_172.jpg', 'airplane_390.jpg', 'airplane_007.jpg', 'airplane_635.jpg', 'airplane_481.jpg', 'airplane_523.jpg', 'airplane_490.jpg', 'airplane_188.jpg', 'airplane_323.jpg', 'airplane_650.jpg', 'airplane_621.jpg', 'airplane_181.jpg', 'airplane_279.jpg', '428501.jpg', 'airplane_328.jpg', 'airplane_397.jpg', 'airplane_497.jpg', 'airplane_164.jpg', 'airplane_658.jpg', 'airplane_222.jpg', 'airplane_101.jpg', 'airplane_178.jpg', 'airplane_121.jpg', 'airplane_516.jpg', 'airplane_480.jpg', 'airplane_014.jpg', 'airplane_476.jpg', 'airplane_119.jpg', 'airplane_286.jpg', 'airplane_668.jpg', 'Planes4.jpg', 'airplane_619.jpg', 'airplane_355.jpg', 'airplane_168.jpg', 'airplane_529.jpg', 'airplane_685.jpg', 'airplane_541.jpg', 'airplane_060.jpg', 'airplane_329.jpg', 'airplane_512.jpg', 'airplane_603.jpg', 'airplane_277.jpg', 'airplane_156.jpg', 'airplane_088.jpg', 'airplane_341.jpg', 'airplane_403.jpg', 'airplane_027.jpg', '42845.jpg', 'airplane_492.jpg', 'airplane_394.jpg', 'airplane_153.jpg', 'airplane_520.jpg', 'airplane_488.jpg', 'airplane_042.jpg', 'airplane_517.jpg', 'airplane_396.jpg', 'airplane_272.jpg', 'airplane_239.jpg', 'airplane_189.jpg', 'airplane_634.jpg']
In [16]:
# checking the airplanes_annotations file present or not
os.listdir("/tmp/Airplanes_Annotations")
Out[16]:
['airplane_132.csv', 'airplane_028.csv', 'airplane_174.csv', 'airplane_280.csv', 'airplane_098.csv', 'airplane_291.csv', 'airplane_250.csv', 'airplane_450.csv', 'airplane_641.csv', '428461.csv', 'airplane_601.csv', 'airplane_417.csv', 'airplane_511.csv', 'airplane_088.csv', 'airplane_540.csv', 'airplane_552.csv', 'airplane_478.csv', 'airplane_684.csv', 'airplane_273.csv', 'airplane_102.csv', 'airplane_089.csv', 'airplane_239.csv', 'airplane_582.csv', 'airplane_126.csv', 'airplane_001.csv', 'airplane_373.csv', 'airplane_683.csv', 'airplane_302.csv', 'airplane_288.csv', 'airplane_482.csv', 'airplane_283.csv', 'airplane_092.csv', 'airplane_496.csv', 'airplane_049.csv', 'airplane_051.csv', 'airplane_418.csv', 'airplane_370.csv', 'airplane_043.csv', 'airplane_257.csv', 'airplane_155.csv', 'airplane_412.csv', 'airplane_016.csv', 'airplane_106.csv', 'airplane_261.csv', 'airplane_650.csv', 'airplane_462.csv', 'airplane_356.csv', 'airplane_036.csv', 'airplane_074.csv', 'airplane_342.csv', 'airplane_402.csv', 'airplane_359.csv', 'airplane_444.csv', 'airplane_260.csv', 'airplane_627.csv', 'airplane_615.csv', 'airplane_640.csv', 'airplane_163.csv', 'airplane_176.csv', 'airplane_500.csv', 'airplane_565.csv', 'airplane_290.csv', 'airplane_385.csv', 'airplane_677.csv', 'airplane_658.csv', 'airplane_292.csv', 'airplane_415.csv', 'airplane_591.csv', 'airplane_052.csv', 'airplane_404.csv', 'airplane_008.csv', 'airplane_507.csv', 'airplane_156.csv', 'airplane_167.csv', 'airplane_399.csv', 'airplane_023.csv', 'airplane_694.csv', 'airplane_100.csv', 'airplane_388.csv', 'airplane_596.csv', 'airplane_390.csv', 'airplane_669.csv', 'airplane_339.csv', 'airplane_224.csv', 'airplane_017.csv', '428472.csv', 'airplane_455.csv', 'airplane_180.csv', 'airplane_631.csv', 'airplane_033.csv', 'airplane_199.csv', 'airplane_371.csv', 'airplane_379.csv', 'airplane_333.csv', 'airplane_245.csv', 'Planes12.csv', 'airplane_560.csv', 'airplane_420.csv', 'airplane_116.csv', 'airplane_697.csv', 'airplane_193.csv', 'airplane_192.csv', 'airplane_699.csv', 'airplane_247.csv', 'airplane_673.csv', 'airplane_410.csv', 'airplane_164.csv', 'airplane_206.csv', 'airplane_686.csv', 'airplane_114.csv', 'airplane_363.csv', 'airplane_029.csv', 'Planes4.csv', 'airplane_593.csv', 'airplane_325.csv', 'airplane_380.csv', 'airplane_396.csv', 'airplane_181.csv', 'airplane_611.csv', 'airplane_295.csv', 'airplane_583.csv', 'airplane_588.csv', 'airplane_360.csv', 'airplane_486.csv', 'airplane_574.csv', 'airplane_354.csv', 'airplane_189.csv', 'airplane_091.csv', 'airplane_213.csv', 'airplane_168.csv', 'airplane_561.csv', 'airplane_009.csv', 'airplane_384.csv', 'airplane_320.csv', 'airplane_458.csv', 'airplane_056.csv', 'airplane_425.csv', 'airplane_604.csv', 'airplane_087.csv', 'airplane_304.csv', 'airplane_665.csv', 'airplane_624.csv', 'airplane_528.csv', 'airplane_476.csv', 'airplane_166.csv', 'airplane_011.csv', 'airplane_125.csv', 'airplane_152.csv', 'airplane_208.csv', 'airplane_136.csv', 'airplane_058.csv', 'airplane_648.csv', 'airplane_309.csv', 'airplane_491.csv', 'airplane_512.csv', 'airplane_622.csv', 'airplane_573.csv', 'airplane_562.csv', 'airplane_634.csv', 'airplane_297.csv', 'airplane_637.csv', 'airplane_055.csv', 'airplane_047.csv', 'airplane_625.csv', 'airplane_223.csv', 'airplane_392.csv', 'airplane_449.csv', 'airplane_436.csv', '428481.csv', 'airplane_696.csv', 'airplane_639.csv', 'airplane_076.csv', 'airplane_662.csv', 'airplane_034.csv', 'airplane_228.csv', 'airplane_348.csv', 'airplane_282.csv', 'airplane_656.csv', 'airplane_653.csv', 'airplane_400.csv', 'airplane_408.csv', 'airplane_518.csv', 'airplane_654.csv', 'airplane_026.csv', 'airplane_267.csv', 'airplane_435.csv', 'airplane_440.csv', 'airplane_494.csv', 'airplane_269.csv', 'airplane_545.csv', 'airplane_147.csv', 'airplane_232.csv', 'airplane_692.csv', 'airplane_660.csv', 'airplane_531.csv', 'airplane_298.csv', 'airplane_693.csv', 'airplane_350.csv', 'airplane_252.csv', 'airplane_585.csv', 'airplane_503.csv', 'airplane_157.csv', 'airplane_383.csv', 'airplane_080.csv', 'airplane_138.csv', 'airplane_521.csv', 'airplane_590.csv', 'airplane_368.csv', 'airplane_603.csv', 'airplane_265.csv', 'airplane_145.csv', 'airplane_119.csv', 'airplane_127.csv', 'airplane_681.csv', 'airplane_312.csv', '428462.csv', 'airplane_063.csv', 'airplane_124.csv', 'airplane_386.csv', 'airplane_465.csv', 'airplane_067.csv', 'airplane_097.csv', 'airplane_534.csv', 'airplane_508.csv', 'airplane_629.csv', 'airplane_216.csv', 'airplane_279.csv', 'airplane_483.csv', 'airplane_556.csv', 'airplane_316.csv', 'airplane_365.csv', 'airplane_430.csv', 'airplane_064.csv', 'airplane_690.csv', 'airplane_587.csv', 'airplane_021.csv', 'airplane_393.csv', 'airplane_141.csv', 'airplane_451.csv', 'airplane_642.csv', 'airplane_096.csv', 'airplane_679.csv', 'airplane_272.csv', 'airplane_249.csv', 'airplane_041.csv', 'airplane_635.csv', 'airplane_129.csv', 'airplane_580.csv', 'airplane_471.csv', 'airplane_278.csv', 'airplane_377.csv', 'airplane_171.csv', 'airplane_190.csv', 'airplane_266.csv', 'airplane_468.csv', 'airplane_479.csv', 'airplane_651.csv', 'airplane_319.csv', 'airplane_346.csv', 'airplane_006.csv', 'airplane_557.csv', 'airplane_217.csv', 'airplane_128.csv', 'airplane_188.csv', 'airplane_169.csv', 'airplane_532.csv', 'hotanairport5.csv', 'airplane_118.csv', 'airplane_432.csv', 'airplane_525.csv', 'airplane_457.csv', 'airplane_446.csv', 'airplane_045.csv', 'airplane_332.csv', '428501.csv', 'airplane_031.csv', 'airplane_025.csv', 'airplane_505.csv', 'airplane_205.csv', 'airplane_427.csv', 'airplane_256.csv', 'airplane_522.csv', 'airplane_243.csv', 'airplane_303.csv', 'airplane_165.csv', 'airplane_572.csv', 'airplane_578.csv', 'airplane_456.csv', 'airplane_463.csv', 'airplane_372.csv', 'airplane_421.csv', 'airplane_276.csv', 'airplane_101.csv', 'airplane_060.csv', 'airplane_002.csv', 'airplane_185.csv', 'airplane_519.csv', 'airplane_459.csv', 'Planes10.csv', 'airplane_554.csv', 'airplane_066.csv', 'airplane_602.csv', 'airplane_082.csv', 'airplane_294.csv', 'airplane_305.csv', 'airplane_134.csv', 'airplane_182.csv', 'airplane_210.csv', 'airplane_434.csv', 'airplane_443.csv', 'airplane_254.csv', 'airplane_081.csv', 'airplane_121.csv', 'airplane_078.csv', 'airplane_110.csv', 'airplane_214.csv', 'airplane_117.csv', 'airplane_311.csv', 'airplane_079.csv', 'airplane_564.csv', 'airplane_135.csv', 'airplane_626.csv', 'airplane_179.csv', 'airplane_584.csv', 'airplane_307.csv', 'airplane_536.csv', 'airplane_003.csv', 'airplane_038.csv', 'airplane_242.csv', 'airplane_086.csv', 'airplane_466.csv', 'airplane_306.csv', 'airplane_559.csv', 'airplane_643.csv', 'airplane_115.csv', 'airplane_546.csv', 'airplane_225.csv', 'airplane_495.csv', 'airplane_233.csv', 'airplane_563.csv', 'airplane_229.csv', 'airplane_170.csv', 'airplane_314.csv', 'hotanairport1-rgb.csv', 'airplane_548.csv', 'airplane_237.csv', 'airplane_010.csv', 'airplane_389.csv', 'airplane_433.csv', 'airplane_070.csv', 'airplane_187.csv', 'airplane_349.csv', 'airplane_403.csv', 'airplane_024.csv', 'airplane_566.csv', 'airplane_146.csv', 'airplane_616.csv', 'airplane_577.csv', 'airplane_544.csv', 'airplane_072.csv', 'airplane_581.csv', '428492.csv', 'airplane_191.csv', 'airplane_313.csv', 'airplane_543.csv', 'airplane_085.csv', 'airplane_514.csv', 'airplane_286.csv', 'airplane_429.csv', 'airplane_004.csv', 'airplane_204.csv', 'airplane_437.csv', 'airplane_020.csv', 'airplane_369.csv', 'airplane_123.csv', 'airplane_186.csv', 'airplane_426.csv', 'Planes9.csv', 'airplane_329.csv', 'airplane_137.csv', 'airplane_226.csv', 'airplane_674.csv', 'airplane_113.csv', 'airplane_178.csv', 'airplane_376.csv', 'airplane_424.csv', 'airplane_133.csv', 'airplane_248.csv', 'airplane_099.csv', 'airplane_594.csv', 'airplane_675.csv', 'airplane_600.csv', 'airplane_553.csv', 'airplane_502.csv', 'airplane_057.csv', 'airplane_484.csv', 'airplane_700.csv', 'airplane_071.csv', 'airplane_069.csv', 'airplane_158.csv', 'airplane_159.csv', 'airplane_353.csv', 'airplane_018.csv', 'airplane_200.csv', 'airplane_240.csv', 'airplane_172.csv', 'airplane_361.csv', 'airplane_084.csv', 'airplane_657.csv', '428491.csv', 'airplane_527.csv', 'airplane_054.csv', 'airplane_567.csv', 'airplane_387.csv', 'airplane_030.csv', 'airplane_007.csv', 'airplane_649.csv', 'airplane_646.csv', 'airplane_447.csv', 'airplane_215.csv', 'airplane_201.csv', 'airplane_691.csv', 'airplane_338.csv', 'airplane_541.csv', 'airplane_469.csv', 'airplane_454.csv', 'airplane_271.csv', 'airplane_606.csv', 'airplane_308.csv', 'airplane_406.csv', 'airplane_062.csv', 'airplane_231.csv', 'airplane_095.csv', 'airplane_613.csv', 'airplane_050.csv', 'airplane_488.csv', 'airplane_139.csv', 'airplane_551.csv', 'airplane_401.csv', 'airplane_414.csv', 'airplane_644.csv', 'airplane_318.csv', 'airplane_340.csv', 'airplane_326.csv', '428483.csv', 'airplane_555.csv', '42847.csv', 'airplane_220.csv', 'airplane_131.csv', 'airplane_607.csv', 'airplane_620.csv', '42845.csv', 'airplane_331.csv', 'airplane_293.csv', 'airplane_140.csv', 'airplane_015.csv', 'Planes3.csv', 'airplane_506.csv', 'airplane_655.csv', 'airplane_301.csv', 'airplane_236.csv', 'airplane_537.csv', 'airplane_274.csv', 'airplane_422.csv', 'airplane_150.csv', 'airplane_682.csv', 'airplane_258.csv', 'airplane_362.csv', 'airplane_529.csv', 'airplane_475.csv', 'airplane_345.csv', 'airplane_411.csv', 'airplane_330.csv', 'airplane_022.csv', 'airplane_211.csv', 'airplane_586.csv', 'airplane_112.csv', 'airplane_235.csv', 'airplane_149.csv', 'airplane_299.csv', 'airplane_398.csv', 'airplane_094.csv', 'airplane_357.csv', 'airplane_428.csv', 'airplane_209.csv', 'airplane_366.csv', 'airplane_287.csv', 'airplane_194.csv', 'airplane_473.csv', 'airplane_222.csv', 'airplane_571.csv', 'airplane_416.csv', 'airplane_623.csv', 'airplane_334.csv', 'airplane_445.csv', 'planes2.csv', 'airplane_664.csv', 'airplane_520.csv', 'airplane_253.csv', 'airplane_284.csv', 'airplane_154.csv', 'airplane_241.csv', 'airplane_219.csv', 'airplane_470.csv', 'airplane_343.csv', 'airplane_409.csv', 'airplane_341.csv', 'airplane_477.csv', 'airplane_275.csv', 'airplane_264.csv', 'airplane_509.csv', '42850.csv', 'airplane_184.csv', 'airplane_542.csv', 'airplane_610.csv', 'hotanairport2-rgb.csv', 'airplane_539.csv', 'airplane_460.csv', 'airplane_448.csv', 'airplane_109.csv', 'airplane_516.csv', 'Planes8.csv', 'airplane_262.csv', 'airplane_504.csv', 'airplane_569.csv', 'airplane_059.csv', 'airplane_668.csv', 'airplane_501.csv', 'airplane_498.csv', 'airplane_685.csv', 'airplane_130.csv', 'airplane_042.csv', 'airplane_538.csv', 'airplane_364.csv', 'airplane_234.csv', 'airplane_688.csv', 'airplane_680.csv', 'airplane_589.csv', 'airplane_321.csv', 'airplane_492.csv', 'airplane_005.csv', 'airplane_570.csv', 'Planes6.csv', 'airplane_027.csv', 'airplane_497.csv', 'airplane_197.csv', 'airplane_238.csv', 'airplane_666.csv', 'airplane_474.csv', 'airplane_281.csv', 'airplane_268.csv', 'airplane_523.csv', 'airplane_351.csv', '428503.csv', 'airplane_659.csv', 'airplane_550.csv', 'airplane_355.csv', 'airplane_107.csv', 'airplane_259.csv', 'airplane_526.csv', 'airplane_595.csv', 'airplane_439.csv', 'airplane_672.csv', 'airplane_676.csv', 'airplane_285.csv', 'airplane_382.csv', 'airplane_144.csv', 'airplane_579.csv', 'airplane_687.csv', 'Planes11.csv', 'airplane_670.csv', 'airplane_441.csv', 'airplane_618.csv', 'airplane_103.csv', 'airplane_335.csv', 'airplane_344.csv', 'airplane_300.csv', 'airplane_296.csv', 'airplane_120.csv', 'airplane_633.csv', 'airplane_324.csv', 'airplane_678.csv', 'airplane_122.csv', 'airplane_044.csv', 'Planes7.csv', 'airplane_612.csv', 'airplane_524.csv', 'airplane_142.csv', 'airplane_472.csv', 'airplane_485.csv', 'airplane_093.csv', 'airplane_196.csv', '428482.csv', 'airplane_251.csv', 'airplane_510.csv', 'airplane_663.csv', 'airplane_619.csv', 'airplane_397.csv', 'airplane_061.csv', 'airplane_162.csv', 'airplane_323.csv', 'airplane_617.csv', 'airplane_391.csv', 'airplane_638.csv', 'airplane_487.csv', 'airplane_549.csv', 'airplane_270.csv', 'airplane_367.csv', 'airplane_513.csv', 'airplane_218.csv', 'airplane_636.csv', 'airplane_547.csv', 'airplane_453.csv', 'airplane_630.csv', 'airplane_143.csv', 'airplane_327.csv', 'airplane_598.csv', 'airplane_032.csv', 'airplane_083.csv', 'airplane_671.csv', 'airplane_490.csv', 'airplane_481.csv', 'airplane_289.csv', 'airplane_019.csv', 'airplane_263.csv', 'airplane_558.csv', 'airplane_230.csv', 'airplane_592.csv', 'airplane_046.csv', 'airplane_077.csv', 'airplane_608.csv', 'airplane_048.csv', 'airplane_244.csv', 'airplane_575.csv', 'airplane_533.csv', 'airplane_177.csv', 'airplane_073.csv', 'airplane_012.csv', 'airplane_053.csv', 'airplane_413.csv', 'airplane_183.csv', 'hotanairport5-rgb.csv', 'airplane_322.csv', 'airplane_419.csv', 'airplane_255.csv', 'airplane_175.csv', 'airplane_035.csv', 'airplane_161.csv', '42848.csv', 'airplane_037.csv', 'airplane_452.csv', 'airplane_576.csv', 'airplane_068.csv', 'airplane_374.csv', 'airplane_647.csv', 'airplane_014.csv', 'airplane_599.csv', 'airplane_203.csv', 'airplane_090.csv', 'airplane_317.csv', 'airplane_698.csv', 'airplane_075.csv', 'airplane_358.csv', 'airplane_221.csv', 'airplane_609.csv', 'airplane_480.csv', 'airplane_407.csv', 'airplane_148.csv', 'airplane_347.csv', 'airplane_515.csv', 'airplane_652.csv', 'airplane_568.csv', 'airplane_336.csv', 'airplane_394.csv', 'airplane_438.csv', 'airplane_153.csv', 'airplane_442.csv', 'airplane_628.csv', 'airplane_105.csv', 'airplane_207.csv', 'airplane_614.csv', 'airplane_227.csv', 'airplane_632.csv', 'airplane_198.csv', 'airplane_111.csv', 'airplane_530.csv', 'airplane_461.csv', 'airplane_040.csv', 'airplane_467.csv', 'airplane_517.csv', 'airplane_464.csv', 'airplane_395.csv', 'airplane_499.csv', 'airplane_160.csv', 'Planes1.csv', 'airplane_212.csv', 'airplane_039.csv', 'airplane_689.csv', 'airplane_661.csv', 'airplane_337.csv', 'airplane_173.csv', 'airplane_202.csv', '428451.csv', 'airplane_431.csv', 'airplane_104.csv', 'airplane_108.csv', 'airplane_315.csv', 'airplane_423.csv', 'airplane_328.csv', 'airplane_489.csv', 'airplane_246.csv', 'airplane_310.csv', 'airplane_195.csv', 'airplane_013.csv', 'airplane_277.csv', 'airplane_645.csv', 'airplane_378.csv', 'airplane_065.csv', 'airplane_352.csv', 'airplane_381.csv', 'airplane_151.csv', 'airplane_405.csv', 'airplane_493.csv', 'airplane_605.csv', 'airplane_621.csv', '42849.csv', 'airplane_695.csv', 'airplane_535.csv', 'airplane_597.csv']
In [17]:
# present working directly
!pwd
/content/aeroplane_detection
In [18]:
# moving the img files from tmp and to the data/images
!mv /tmp/Images/* data/images
In [19]:
# moving the annotaion from tmp and to data/train labels
!mv /tmp/Airplanes_Annotations/* data/train_labels
In [20]:
# lists the files inside 'annotations' in a random order (not really random, by their hash value instead)
# Moves the first 200 labels to the testing dir: `test_labels`
!ls data/train_labels/* | sort -R | head -200 | xargs -I{} mv {} data/test_labels
In [21]:
# to divide into train labels into 529
!ls data/train_labels/ | wc -l
529
In [22]:
# to divide into the test labesl
!ls data/test_labels/ | wc -l
200
In [23]:
# present working directory
!pwd
/content/aeroplane_detection
In [24]:
# changing the directory to data
cd data
/content/aeroplane_detection/data
In [25]:
# present working directory
!pwd
/content/aeroplane_detection/data
In [26]:
# changing the directory to train_labels
cd train_labels
/content/aeroplane_detection/data/train_labels
In [27]:
# logic to convert it into list and data frame train
train=[]
import os
for f in os.listdir():
file1=open(f,'r')
for line in file1.readlines():
line=line.strip('\n')
l=[str(f.split('.')[0])+".jpg"]
l=l+line.split(' ')
l.append('aeroplane')
train.append(l)
print(len(train))
1965
In [28]:
# going back to existing workijng directory
cd -
/content/aeroplane_detection/data
In [29]:
# changing the working directory
cd test_labels
/content/aeroplane_detection/data/test_labels
In [30]:
# checking the present directory
!pwd
/content/aeroplane_detection/data/test_labels
In [31]:
# logic to convert it into list and data frame test
test=[]
import os
for f in os.listdir():
file1=open(f,'r')
for line in file1.readlines():
line=line.strip('\n')
l=[str(f.split('.')[0])+".jpg"]
l=l+line.split(' ')
l.append('aeroplane')
test.append(l)
print(len(test))
942
In [32]:
# assigning the column names for the data frame train
import pandas as pd
train=pd.DataFrame(train,columns=['filename','xmin','ymin','xmax','ymax','class'])
In [33]:
# assigning the column names for the data frame test
test=pd.DataFrame(test,columns=['filename','xmin','ymin','xmax','ymax','class'])
In [34]:
# traindata frame
test
Out[34]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 0 | 428461.jpg | 176 | 12 | 207 | 41 | aeroplane |
| 1 | 428461.jpg | 155 | 47 | 188 | 82 | aeroplane |
| 2 | 428461.jpg | 86 | 125 | 118 | 160 | aeroplane |
| 3 | 428461.jpg | 54 | 108 | 83 | 134 | aeroplane |
| 4 | 428461.jpg | 0 | 65 | 22 | 104 | aeroplane |
| ... | ... | ... | ... | ... | ... | ... |
| 937 | airplane_381.jpg | 46 | 79 | 135 | 141 | aeroplane |
| 938 | airplane_381.jpg | 178 | 30 | 254 | 107 | aeroplane |
| 939 | airplane_621.jpg | 93 | 33 | 173 | 93 | aeroplane |
| 940 | airplane_621.jpg | 40 | 94 | 132 | 161 | aeroplane |
| 941 | airplane_621.jpg | 23 | 180 | 83 | 236 | aeroplane |
942 rows × 6 columns
In [35]:
# train data frame
train
Out[35]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 0 | airplane_132.jpg | 59 | 163 | 99 | 205 | aeroplane |
| 1 | airplane_132.jpg | 100 | 78 | 117 | 97 | aeroplane |
| 2 | airplane_132.jpg | 115 | 61 | 133 | 81 | aeroplane |
| 3 | airplane_132.jpg | 223 | 104 | 256 | 157 | aeroplane |
| 4 | airplane_132.jpg | 219 | 3 | 255 | 19 | aeroplane |
| ... | ... | ... | ... | ... | ... | ... |
| 1960 | airplane_695.jpg | 8 | 11 | 249 | 255 | aeroplane |
| 1961 | airplane_535.jpg | 167 | 138 | 221 | 191 | aeroplane |
| 1962 | airplane_597.jpg | 2 | 204 | 43 | 249 | aeroplane |
| 1963 | airplane_597.jpg | 163 | 105 | 212 | 144 | aeroplane |
| 1964 | airplane_597.jpg | 225 | 72 | 253 | 120 | aeroplane |
1965 rows × 6 columns
In [36]:
# checking the missing values for train
train.isnull().sum()
Out[36]:
filename 0 xmin 0 ymin 0 xmax 1 ymax 1 class 1 dtype: int64
In [37]:
# checking the missing values for test
train.isnull().sum()
Out[37]:
filename 0 xmin 0 ymin 0 xmax 1 ymax 1 class 1 dtype: int64
In [39]:
# missing values in the data frame in train
train[train['ymin']=='aeroplane']
Out[39]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 1282 | airplane_112.jpg | 8 | aeroplane | None | None | None |
In [40]:
# droping
train.drop([1282],axis=0,inplace=True)
In [41]:
# missing values in the data frame in test
test[test['ymin']=='aeroplane']
Out[41]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 402 | airplane_287.jpg | 2 | aeroplane | None | None | None |
In [42]:
# droping
test.drop([402],axis=0,inplace=True)
In [43]:
# checking if any missing values
train[train['ymin']=='aeroplane']
Out[43]:
| filename | xmin | ymin | xmax | ymax | class |
|---|
In [44]:
# checking if any missing values
test[test['ymin']=='aeroplane']
Out[44]:
| filename | xmin | ymin | xmax | ymax | class |
|---|
In [45]:
# converting to.csv files
train.to_csv('/content/aeroplane_detection/data/train_labels.csv',index=False)
In [46]:
# converting to.csv files
test.to_csv('/content/aeroplane_detection/data/test_labels.csv',index=False)
In [47]:
# present working directory
!pwd
/content/aeroplane_detection/data/test_labels
In [48]:
# checking the train.csv data file
train_df=pd.read_csv('/content/aeroplane_detection/data/train_labels.csv')
train_df
Out[48]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 0 | airplane_132.jpg | 59 | 163 | 99 | 205 | aeroplane |
| 1 | airplane_132.jpg | 100 | 78 | 117 | 97 | aeroplane |
| 2 | airplane_132.jpg | 115 | 61 | 133 | 81 | aeroplane |
| 3 | airplane_132.jpg | 223 | 104 | 256 | 157 | aeroplane |
| 4 | airplane_132.jpg | 219 | 3 | 255 | 19 | aeroplane |
| ... | ... | ... | ... | ... | ... | ... |
| 1959 | airplane_695.jpg | 8 | 11 | 249 | 255 | aeroplane |
| 1960 | airplane_535.jpg | 167 | 138 | 221 | 191 | aeroplane |
| 1961 | airplane_597.jpg | 2 | 204 | 43 | 249 | aeroplane |
| 1962 | airplane_597.jpg | 163 | 105 | 212 | 144 | aeroplane |
| 1963 | airplane_597.jpg | 225 | 72 | 253 | 120 | aeroplane |
1964 rows × 6 columns
In [49]:
# checking the test.csv data file
test_df=pd.read_csv('/content/aeroplane_detection/data/test_labels.csv')
test_df.head()
Out[49]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 0 | 428461.jpg | 176 | 12 | 207 | 41 | aeroplane |
| 1 | 428461.jpg | 155 | 47 | 188 | 82 | aeroplane |
| 2 | 428461.jpg | 86 | 125 | 118 | 160 | aeroplane |
| 3 | 428461.jpg | 54 | 108 | 83 | 134 | aeroplane |
| 4 | 428461.jpg | 0 | 65 | 22 | 104 | aeroplane |
In [50]:
# present working directory
!pwd
/content/aeroplane_detection/data/test_labels
In [51]:
# changing the directory
cd -
/content/aeroplane_detection/data
In [53]:
# display class files as list
%cd /content/aeroplane_detection/data
label_map_path = os.path.join("label_map.pbtxt")
classes=['aeroplane']
pbtxt_content = ""
print(type(classes))
#creats a pbtxt file the has the class names.
for i, class_name in enumerate(classes):
# display_name is optional.
pbtxt_content = (
pbtxt_content
+ "item {{\n id: {0}\n name: '{1}'\n display_name: 'aeroplane'\n }}\n\n".format(i + 1, class_name)
)
pbtxt_content = pbtxt_content.strip()
with open(label_map_path, "w") as f:
f.write(pbtxt_content)
/content/aeroplane_detection/data <class 'list'>
In [54]:
#checking the pbtxt file
!cat label_map.pbtxt
item {
id: 1
name: 'aeroplane'
display_name: 'aeroplane'
}
In [55]:
# checking the files
!ls -l
total 200 drwxr-xr-x 2 root root 32768 Jul 25 06:23 images -rw-r--r-- 1 root root 71 Jul 25 06:27 label_map.pbtxt drwxr-xr-x 2 root root 12288 Jul 25 06:23 test_labels -rw-r--r-- 1 root root 37671 Jul 25 06:26 test_labels.csv drwxr-xr-x 2 root root 32768 Jul 25 06:23 train_labels -rw-r--r-- 1 root root 79564 Jul 25 06:26 train_labels.csv
In [56]:
# present working dirrectory
!pwd
/content/aeroplane_detection/data
In [58]:
# converting the train and test data frames to _csv
%cd /content/aeroplane_detection/data
train.to_csv('train_labels.csv', encoding='utf-8', index=False)
test.to_csv('test_labels.csv', encoding='utf-8', index=False)
/content/aeroplane_detection/data
In [59]:
# checking the files
!ls -l
total 200 drwxr-xr-x 2 root root 32768 Jul 25 06:23 images -rw-r--r-- 1 root root 71 Jul 25 06:27 label_map.pbtxt drwxr-xr-x 2 root root 12288 Jul 25 06:23 test_labels -rw-r--r-- 1 root root 37671 Jul 25 06:28 test_labels.csv drwxr-xr-x 2 root root 32768 Jul 25 06:23 train_labels -rw-r--r-- 1 root root 79564 Jul 25 06:28 train_labels.csv
In [60]:
# Downlaods Tenorflow
%cd /content/aeroplane_detection/
!git clone --q https://github.com/tensorflow/models.git
/content/aeroplane_detection
In [61]:
# present working directory
!pwd
/content/aeroplane_detection
In [62]:
# moving the models/officical to models/research/officcial
!mv models/official models/research/official
In [63]:
%cd /content/aeroplane_detection/models/research
#compiling the proto buffers (not important to understand for this project but you can learn more about them here: https://developers.google.com/protocol-buffers/)
!protoc object_detection/protos/*.proto --python_out=.
# exports the PYTHONPATH environment variable with the reasearch and slim folders' paths
os.environ['PYTHONPATH'] += ':/content/aeroplane_detection/models/research/:/content/aeroplane_detection/models/research/slim/'
/content/aeroplane_detection/models/research object_detection/protos/input_reader.proto: warning: Import object_detection/protos/image_resizer.proto but not used.
In [64]:
# testing the model builder
!pip install tf_slim
!python3 object_detection/builders/model_builder_test.py
Collecting tf_slim
Downloading https://files.pythonhosted.org/packages/02/97/b0f4a64df018ca018cc035d44f2ef08f91e2e8aa67271f6f19633a015ff7/tf_slim-1.1.0-py2.py3-none-any.whl (352kB)
|████████████████████████████████| 358kB 3.5MB/s
Requirement already satisfied: absl-py>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from tf_slim) (0.9.0)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from absl-py>=0.2.2->tf_slim) (1.15.0)
Installing collected packages: tf-slim
Successfully installed tf-slim-1.1.0
In [65]:
test[test['ymin']=='aeroplane']
Out[65]:
| filename | xmin | ymin | xmax | ymax | class |
|---|
In [66]:
train[train['ymin']=='aeroplane']
Out[66]:
| filename | xmin | ymin | xmax | ymax | class |
|---|
In [67]:
#adjusted from: https://github.com/datitran/raccoon_dataset
# converts the csv files for training and testing data to two TFRecords files.
# places the output in the same directory as the input
from object_detection.utils import dataset_util
%cd /content/aeroplane_detection/models/
DATA_BASE_PATH = '/content/aeroplane_detection/data/'
image_dir = DATA_BASE_PATH +'images/'
def class_text_to_int(row_label):
if row_label == 'aeroplane':
return 1
else:
None
def split(df, group):
data = namedtuple('data', ['filename', 'object'])
gb = df.groupby(group)
return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]
def create_tf_example(group, path):
with tf.io.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = Image.open(encoded_jpg_io)
width, height = image.size
filename = group.filename.encode('utf8')
image_format = b'jpg'
xmins = []
xmaxs = []
ymins = []
ymaxs = []
classes_text = []
classes = []
for index, row in group.object.iterrows():
xmins.append(row['xmin'] / width)
xmaxs.append(row['xmax'] / width)
ymins.append(row['ymin'] / height)
ymaxs.append(row['ymax'] / height)
classes_text.append(row['class'].encode('utf8'))
classes.append(class_text_to_int(row['class']))
tf_example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(height),
'image/width': dataset_util.int64_feature(width),
'image/filename': dataset_util.bytes_feature(filename),
'image/source_id': dataset_util.bytes_feature(filename),
'image/encoded': dataset_util.bytes_feature(encoded_jpg),
'image/format': dataset_util.bytes_feature(image_format),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
}))
return tf_example
for csv in ['train_labels', 'test_labels']:
writer = tf.io.TFRecordWriter(DATA_BASE_PATH + csv + '.record')
path = os.path.join(image_dir)
examples = pd.read_csv(DATA_BASE_PATH + csv + '.csv')
grouped = split(examples, 'filename')
for group in grouped:
tf_example = create_tf_example(group, path)
writer.write(tf_example.SerializeToString())
writer.close()
output_path = os.path.join(os.getcwd(), DATA_BASE_PATH + csv + '.record')
print('Successfully created the TFRecords: {}'.format(DATA_BASE_PATH +csv + '.record'))
/content/aeroplane_detection/models Successfully created the TFRecords: /content/aeroplane_detection/data/train_labels.record Successfully created the TFRecords: /content/aeroplane_detection/data/test_labels.record
In [69]:
# checking whether the boundary box fixed or not
%cd /content/aeroplane_detection/data
from PIL import Image
os.listdir('images')
#image = Image.open("images/4f30dc4b8e2a583d.jpg")
#height,width=image.size
#print(height,width)
import matplotlib.pyplot as plt
import matplotlib.patches as patches
for row in train.values[1:]:
img = plt.imread("images/"+row[0])
plt.imshow(img)
print(row[0],row[2],row[3],row[4])
plt.plot([float(row[1]),float(row[3]),float(row[3]),float(row[1]),float(row[1])],[float(row[4]),float(row[4]),float(row[2]),float(row[2]),float(row[4])],color='r')
break
/content/aeroplane_detection/data airplane_132.jpg 78 117 97
In [70]:
examples
Out[70]:
| filename | xmin | ymin | xmax | ymax | class | |
|---|---|---|---|---|---|---|
| 0 | 428461.jpg | 176 | 12 | 207 | 41 | aeroplane |
| 1 | 428461.jpg | 155 | 47 | 188 | 82 | aeroplane |
| 2 | 428461.jpg | 86 | 125 | 118 | 160 | aeroplane |
| 3 | 428461.jpg | 54 | 108 | 83 | 134 | aeroplane |
| 4 | 428461.jpg | 0 | 65 | 22 | 104 | aeroplane |
| ... | ... | ... | ... | ... | ... | ... |
| 936 | airplane_381.jpg | 46 | 79 | 135 | 141 | aeroplane |
| 937 | airplane_381.jpg | 178 | 30 | 254 | 107 | aeroplane |
| 938 | airplane_621.jpg | 93 | 33 | 173 | 93 | aeroplane |
| 939 | airplane_621.jpg | 40 | 94 | 132 | 161 | aeroplane |
| 940 | airplane_621.jpg | 23 | 180 | 83 | 236 | aeroplane |
941 rows × 6 columns
In [71]:
# TFRecords are created
!ls -lX /content/aeroplane_detection/data/
total 9324 drwxr-xr-x 2 root root 32768 Jul 25 06:23 images drwxr-xr-x 2 root root 12288 Jul 25 06:23 test_labels drwxr-xr-x 2 root root 32768 Jul 25 06:23 train_labels -rw-r--r-- 1 root root 37671 Jul 25 06:28 test_labels.csv -rw-r--r-- 1 root root 79564 Jul 25 06:28 train_labels.csv -rw-r--r-- 1 root root 71 Jul 25 06:27 label_map.pbtxt -rw-r--r-- 1 root root 2563245 Jul 25 06:30 test_labels.record -rw-r--r-- 1 root root 6775190 Jul 25 06:30 train_labels.record
In [72]:
# downloading the base model
%cd /content/aeroplane_detection/models/research
# Name of the object detection model to use.
MODEL = MODELS_CONFIG[selected_model]['model_name']
# Name of the pipline file in tensorflow object detection API.
pipeline_file = MODELS_CONFIG[selected_model]['pipeline_file']
#selecting the model
MODEL_FILE = MODEL + '.tar.gz'
#creating the downlaod link for the model selected
DOWNLOAD_BASE = 'http://download.tensorflow.org/models/object_detection/'
#the distination folder where the model will be saved
fine_tune_dir = '/content/aeroplane_detection/models/research/pretrained_model'
#checks if the model has already been downloaded
if not (os.path.exists(MODEL_FILE)):
urllib.request.urlretrieve(DOWNLOAD_BASE + MODEL_FILE, MODEL_FILE)
#unzipping the file and extracting its content
tar = tarfile.open(MODEL_FILE)
tar.extractall()
tar.close()
# creating an output file to save the model while training
os.remove(MODEL_FILE)
if (os.path.exists(fine_tune_dir)):
shutil.rmtree(fine_tune_dir)
os.rename(MODEL, fine_tune_dir)
/content/aeroplane_detection/models/research
In [73]:
#checking the content of the pretrained model.
# this is the directory of the "fine_tune_checkpoint" that is used in the config file.
!echo {fine_tune_dir}
!ls -alh {fine_tune_dir}
/content/aeroplane_detection/models/research/pretrained_model total 135M drwxr-xr-x 3 345018 89939 4.0K Mar 30 2018 . drwxr-xr-x 64 root root 4.0K Jul 25 06:32 .. -rw-r--r-- 1 345018 89939 77 Mar 30 2018 checkpoint -rw-r--r-- 1 345018 89939 67M Mar 30 2018 frozen_inference_graph.pb -rw-r--r-- 1 345018 89939 65M Mar 30 2018 model.ckpt.data-00000-of-00001 -rw-r--r-- 1 345018 89939 15K Mar 30 2018 model.ckpt.index -rw-r--r-- 1 345018 89939 3.4M Mar 30 2018 model.ckpt.meta -rw-r--r-- 1 345018 89939 4.2K Mar 30 2018 pipeline.config drwxr-xr-x 3 345018 89939 4.0K Mar 30 2018 saved_model
In [74]:
# configering the training the pipe line
#the path to the folder containing all the sample config files
CONFIG_BASE = "/content/aeroplane_detection/models/research/object_detection/samples/configs/"
#path to the specified model's config file
model_pipline = os.path.join(CONFIG_BASE, pipeline_file)
model_pipline
Out[74]:
'/content/aeroplane_detection/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config'
In [75]:
#editing the configuration file to add the path for the TFRecords files, pbtxt,batch_size,num_steps,num_classes.
# any image augmentation, hyperparemeter tunning (drop out, batch normalization... etc) would be editted here
%%writefile {model_pipline}
model {
ssd {
num_classes: 1 # number of classes to be detected
box_coder {
faster_rcnn_box_coder {
y_scale: 10.0
x_scale: 10.0
height_scale: 5.0
width_scale: 5.0
}
}
matcher {
argmax_matcher {
matched_threshold: 0.5
unmatched_threshold: 0.5
ignore_thresholds: false
negatives_lower_than_unmatched: true
force_match_for_each_row: true
}
}
similarity_calculator {
iou_similarity {
}
}
anchor_generator {
ssd_anchor_generator {
num_layers: 6
min_scale: 0.2
max_scale: 0.95
aspect_ratios: 1.0
aspect_ratios: 2.0
aspect_ratios: 0.5
aspect_ratios: 3.0
aspect_ratios: 0.3333
}
}
# all images will be resized to the below W x H.
image_resizer {
fixed_shape_resizer {
height: 300
width: 300
}
}
box_predictor {
convolutional_box_predictor {
min_depth: 0
max_depth: 0
num_layers_before_predictor: 0
#use_dropout: false
use_dropout: true # to counter over fitting. you can also try tweaking its probability below
dropout_keep_probability: 0.8
kernel_size: 1
box_code_size: 4
apply_sigmoid_to_scores: false
conv_hyperparams {
activation: RELU_6,
regularizer {
l2_regularizer {
# weight: 0.00004
weight: 0.001 # higher regularizition to counter overfitting
}
}
initializer {
truncated_normal_initializer {
stddev: 0.03
mean: 0.0
}
}
batch_norm {
train: true,
scale: true,
center: true,
decay: 0.9997,
epsilon: 0.001,
}
}
}
}
feature_extractor {
type: 'ssd_mobilenet_v2'
min_depth: 16
depth_multiplier: 1.0
conv_hyperparams {
activation: RELU_6,
regularizer {
l2_regularizer {
# weight: 0.00004
weight: 0.001 # higher regularizition to counter overfitting
}
}
initializer {
truncated_normal_initializer {
stddev: 0.03
mean: 0.0
}
}
batch_norm {
train: true,
scale: true,
center: true,
decay: 0.9997,
epsilon: 0.001,
}
}
}
loss {
classification_loss {
weighted_sigmoid {
}
}
localization_loss {
weighted_smooth_l1 {
}
}
hard_example_miner {
num_hard_examples: 3000
iou_threshold: 0.95
loss_type: CLASSIFICATION
max_negatives_per_positive: 3
min_negatives_per_image: 3
}
classification_weight: 1.0
localization_weight: 1.0
}
normalize_loss_by_num_matches: true
post_processing {
batch_non_max_suppression {
score_threshold: 1e-8
iou_threshold: 0.6
#adjust this to the max number of objects per class.
# ex, in my case, i have one pistol in most of the images.
# . there are some images with more than one up to 16.
max_detections_per_class: 16
# max number of detections among all classes. I have 1 class only so
max_total_detections: 16
}
score_converter: SIGMOID
}
}
}
train_config: {
batch_size: 16 # training batch size
optimizer {
rms_prop_optimizer: {
learning_rate: {
exponential_decay_learning_rate {
initial_learning_rate: 0.003
decay_steps: 800720
decay_factor: 0.95
}
}
momentum_optimizer_value: 0.9
decay: 0.9
epsilon: 1.0
}
}
#the path to the pretrained model.
fine_tune_checkpoint: "/content/aeroplane_detection/models/research/pretrained_model/model.ckpt"
fine_tune_checkpoint_type: "detection"
# Note: The below line limits the training process to 200K steps, which we
# empirically found to be sufficient enough to train the pets dataset. This
# effectively bypasses the learning rate schedule (the learning rate will
# never decay). Remove the below line to train indefinitely.
num_steps:50000
#data augmentaion is done here, you can remove or add more.
# They will help the model generalize but the training time will increase greatly by using more data augmentation.
# Check this link to add more image augmentation: https://github.com/tensorflow/models/blob/master/research/object_detection/protos/preprocessor.proto
data_augmentation_options {
random_horizontal_flip {
}
}
data_augmentation_options {
random_adjust_contrast {
}
}
data_augmentation_options {
ssd_random_crop {
}
}
}
train_input_reader: {
tf_record_input_reader {
#path to the training TFRecord
input_path: "/content/aeroplane_detection/data/train_labels.record"
}
#path to the label map
label_map_path: "/content/aeroplane_detection/data/label_map.pbtxt"
}
eval_config: {
# the number of images in your "testing" data (was 200 but we removed one above :) )
num_examples: 200
# the number of images to disply in Tensorboard while training
num_visualizations: 20
# Note: The below line limits the evaluation process to 10 evaluations.
# Remove the below line to evaluate indefinitely.
#max_evals: 10
}
eval_input_reader: {
tf_record_input_reader {
#path to the testing TFRecord
input_path: "/content/aeroplane_detection/data/test_labels.record"
}
#path to the label map
label_map_path: "/content/aeroplane_detection/data/label_map.pbtxt"
shuffle: false
num_readers: 1
}
Overwriting /content/aeroplane_detection/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config
In [76]:
# present working directory
!pwd
/content/aeroplane_detection/models/research
In [77]:
# where the model will be saved at each checkpoint while training
model_dir = 'training/'
# Optionally: remove content in output model directory to fresh start.
!rm -rf {model_dir}
os.makedirs(model_dir, exist_ok=True)
model_dir
Out[77]:
'training/'
In [78]:
#downlaoding ngrok to be able to access tensorboard on google colab
!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
!unzip -o ngrok-stable-linux-amd64.zip
--2020-07-25 06:33:50-- https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip Resolving bin.equinox.io (bin.equinox.io)... 54.80.22.251, 54.88.63.64, 52.6.123.150, ... Connecting to bin.equinox.io (bin.equinox.io)|54.80.22.251|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 13773305 (13M) [application/octet-stream] Saving to: ‘ngrok-stable-linux-amd64.zip’ ngrok-stable-linux- 100%[===================>] 13.13M 40.7MB/s in 0.3s 2020-07-25 06:33:50 (40.7 MB/s) - ‘ngrok-stable-linux-amd64.zip’ saved [13773305/13773305] Archive: ngrok-stable-linux-amd64.zip inflating: ngrok
In [79]:
#the logs that are created while training
LOG_DIR = model_dir
get_ipython().system_raw(
'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'
.format(LOG_DIR)
)
get_ipython().system_raw('./ngrok http 6006 &')
In [80]:
#The link to tensorboard.
#works after the training starts.
### note: if you didnt get a link as output, rerun this cell and the one above
!curl -s http://localhost:4040/api/tunnels | python3 -c \
"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"
http://bfa0563666a3.ngrok.io
In [81]:
# checking the path pipline
model_pipline
Out[81]:
'/content/aeroplane_detection/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config'
In [82]:
# checking the directory path
model_dir
Out[82]:
'training/'
In [83]:
# present working directory
!pwd
/content/aeroplane_detection/models/research
TRAINING¶
In [84]:
model_dir
Out[84]:
'training/'
In [85]:
model_pipline
Out[85]:
'/content/aeroplane_detection/models/research/object_detection/samples/configs/ssd_mobilenet_v2_coco.config'
In [86]:
!python3 /content/aeroplane_detection/models/research/object_detection/model_main.py \
--pipeline_config_path={model_pipline}\
--model_dir={model_dir} \
--alsologtostderr \
Streaming output truncated to the last 5000 lines.
I0725 07:36:44.348368 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 07:36:44.349223: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:36:44.349828: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 07:36:44.349929: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 07:36:44.349990: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 07:36:44.350085: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 07:36:44.350156: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 07:36:44.350216: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 07:36:44.350289: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 07:36:44.350350: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 07:36:44.350517: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:36:44.351114: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:36:44.351593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 07:36:44.351651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 07:36:44.351675: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 07:36:44.351702: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 07:36:44.351917: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:36:44.352521: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:36:44.353094: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-6887
I0725 07:36:44.354381 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-6887
INFO:tensorflow:Running local_init_op.
I0725 07:36:45.285002 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 07:36:45.423975 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 07:36:56.475315 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 07:36:56.476796 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 07:36:56.480424 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.09s).
Accumulating evaluation results...
DONE (t=0.09s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.001
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.001
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.002
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.003
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.072
INFO:tensorflow:Finished evaluation at 2020-07-25-07:36:57
I0725 07:36:57.759746 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-07:36:57
INFO:tensorflow:Saving dict for global step 6887: DetectionBoxes_Precision/mAP = 0.00018640574, DetectionBoxes_Precision/mAP (large) = 0.0008583396, DetectionBoxes_Precision/mAP (medium) = 0.0, DetectionBoxes_Precision/mAP (small) = 0.0, DetectionBoxes_Precision/mAP@.50IOU = 0.00090009003, DetectionBoxes_Precision/mAP@.75IOU = 1.4194968e-05, DetectionBoxes_Recall/AR@1 = 0.00023952096, DetectionBoxes_Recall/AR@10 = 0.0020359282, DetectionBoxes_Recall/AR@100 = 0.00251497, DetectionBoxes_Recall/AR@100 (large) = 0.072413795, DetectionBoxes_Recall/AR@100 (medium) = 0.0, DetectionBoxes_Recall/AR@100 (small) = 0.0, Loss/classification_loss = 9.632383, Loss/localization_loss = 4.1506624, Loss/regularization_loss = 0.84412074, Loss/total_loss = 14.627167, global_step = 6887, learning_rate = 0.003, loss = 14.627167
I0725 07:36:57.760130 139696487446400 estimator.py:2049] Saving dict for global step 6887: DetectionBoxes_Precision/mAP = 0.00018640574, DetectionBoxes_Precision/mAP (large) = 0.0008583396, DetectionBoxes_Precision/mAP (medium) = 0.0, DetectionBoxes_Precision/mAP (small) = 0.0, DetectionBoxes_Precision/mAP@.50IOU = 0.00090009003, DetectionBoxes_Precision/mAP@.75IOU = 1.4194968e-05, DetectionBoxes_Recall/AR@1 = 0.00023952096, DetectionBoxes_Recall/AR@10 = 0.0020359282, DetectionBoxes_Recall/AR@100 = 0.00251497, DetectionBoxes_Recall/AR@100 (large) = 0.072413795, DetectionBoxes_Recall/AR@100 (medium) = 0.0, DetectionBoxes_Recall/AR@100 (small) = 0.0, Loss/classification_loss = 9.632383, Loss/localization_loss = 4.1506624, Loss/regularization_loss = 0.84412074, Loss/total_loss = 14.627167, global_step = 6887, learning_rate = 0.003, loss = 14.627167
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 6887: training/model.ckpt-6887
I0725 07:36:57.764624 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 6887: training/model.ckpt-6887
INFO:tensorflow:global_step/sec: 1.3913
I0725 07:37:04.921498 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3913
INFO:tensorflow:loss = 2.897071, step = 6900 (71.875 sec)
I0725 07:37:04.922692 139696487446400 basic_session_run_hooks.py:260] loss = 2.897071, step = 6900 (71.875 sec)
INFO:tensorflow:global_step/sec: 1.97524
I0725 07:37:55.548321 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97524
INFO:tensorflow:loss = 3.7464807, step = 7000 (50.627 sec)
I0725 07:37:55.549511 139696487446400 basic_session_run_hooks.py:260] loss = 3.7464807, step = 7000 (50.627 sec)
INFO:tensorflow:global_step/sec: 1.98191
I0725 07:38:46.004909 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98191
INFO:tensorflow:loss = 3.955777, step = 7100 (50.457 sec)
I0725 07:38:46.006052 139696487446400 basic_session_run_hooks.py:260] loss = 3.955777, step = 7100 (50.457 sec)
INFO:tensorflow:global_step/sec: 1.98112
I0725 07:39:36.481377 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98112
INFO:tensorflow:loss = 2.9643884, step = 7200 (50.477 sec)
I0725 07:39:36.482669 139696487446400 basic_session_run_hooks.py:260] loss = 2.9643884, step = 7200 (50.477 sec)
INFO:tensorflow:global_step/sec: 1.97632
I0725 07:40:27.080467 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97632
INFO:tensorflow:loss = 3.4304357, step = 7300 (50.599 sec)
I0725 07:40:27.081690 139696487446400 basic_session_run_hooks.py:260] loss = 3.4304357, step = 7300 (50.599 sec)
INFO:tensorflow:global_step/sec: 1.95916
I0725 07:41:18.122941 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95916
INFO:tensorflow:loss = 4.1768036, step = 7400 (51.043 sec)
I0725 07:41:18.124303 139696487446400 basic_session_run_hooks.py:260] loss = 4.1768036, step = 7400 (51.043 sec)
INFO:tensorflow:global_step/sec: 1.97145
I0725 07:42:08.847083 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97145
INFO:tensorflow:loss = 3.1377733, step = 7500 (50.724 sec)
I0725 07:42:08.848434 139696487446400 basic_session_run_hooks.py:260] loss = 3.1377733, step = 7500 (50.724 sec)
INFO:tensorflow:global_step/sec: 1.9672
I0725 07:42:59.680727 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9672
INFO:tensorflow:loss = 4.1468697, step = 7600 (50.834 sec)
I0725 07:42:59.682165 139696487446400 basic_session_run_hooks.py:260] loss = 4.1468697, step = 7600 (50.834 sec)
INFO:tensorflow:global_step/sec: 1.96955
I0725 07:43:50.453878 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96955
INFO:tensorflow:loss = 3.8476577, step = 7700 (50.773 sec)
I0725 07:43:50.455334 139696487446400 basic_session_run_hooks.py:260] loss = 3.8476577, step = 7700 (50.773 sec)
INFO:tensorflow:global_step/sec: 1.9548
I0725 07:44:41.610007 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9548
INFO:tensorflow:loss = 4.410284, step = 7800 (51.156 sec)
I0725 07:44:41.611315 139696487446400 basic_session_run_hooks.py:260] loss = 4.410284, step = 7800 (51.156 sec)
INFO:tensorflow:global_step/sec: 1.96414
I0725 07:45:32.523008 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96414
INFO:tensorflow:loss = 4.6151643, step = 7900 (50.913 sec)
I0725 07:45:32.524763 139696487446400 basic_session_run_hooks.py:260] loss = 4.6151643, step = 7900 (50.913 sec)
INFO:tensorflow:global_step/sec: 1.96016
I0725 07:46:23.539203 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96016
INFO:tensorflow:loss = 3.520031, step = 8000 (51.016 sec)
I0725 07:46:23.540550 139696487446400 basic_session_run_hooks.py:260] loss = 3.520031, step = 8000 (51.016 sec)
INFO:tensorflow:Saving checkpoints for 8027 into training/model.ckpt.
I0725 07:46:36.655113 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 8027 into training/model.ckpt.
INFO:tensorflow:Skip the current checkpoint eval due to throttle secs (600 secs).
I0725 07:46:38.176026 139696487446400 training.py:527] Skip the current checkpoint eval due to throttle secs (600 secs).
INFO:tensorflow:global_step/sec: 1.92132
I0725 07:47:15.586792 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.92132
INFO:tensorflow:loss = 3.5592806, step = 8100 (52.048 sec)
I0725 07:47:15.588140 139696487446400 basic_session_run_hooks.py:260] loss = 3.5592806, step = 8100 (52.048 sec)
INFO:tensorflow:global_step/sec: 1.97148
I0725 07:48:06.310191 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97148
INFO:tensorflow:loss = 3.9287515, step = 8200 (50.723 sec)
I0725 07:48:06.311379 139696487446400 basic_session_run_hooks.py:260] loss = 3.9287515, step = 8200 (50.723 sec)
INFO:tensorflow:global_step/sec: 1.96554
I0725 07:48:57.186689 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96554
INFO:tensorflow:loss = 3.6700094, step = 8300 (50.877 sec)
I0725 07:48:57.188173 139696487446400 basic_session_run_hooks.py:260] loss = 3.6700094, step = 8300 (50.877 sec)
INFO:tensorflow:global_step/sec: 1.97115
I0725 07:49:47.918554 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97115
INFO:tensorflow:loss = 3.0773747, step = 8400 (50.732 sec)
I0725 07:49:47.920077 139696487446400 basic_session_run_hooks.py:260] loss = 3.0773747, step = 8400 (50.732 sec)
INFO:tensorflow:global_step/sec: 1.98148
I0725 07:50:38.385961 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98148
INFO:tensorflow:loss = 4.099184, step = 8500 (50.467 sec)
I0725 07:50:38.387528 139696487446400 basic_session_run_hooks.py:260] loss = 4.099184, step = 8500 (50.467 sec)
INFO:tensorflow:global_step/sec: 1.9871
I0725 07:51:28.710404 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9871
INFO:tensorflow:loss = 3.4044304, step = 8600 (50.324 sec)
I0725 07:51:28.711556 139696487446400 basic_session_run_hooks.py:260] loss = 3.4044304, step = 8600 (50.324 sec)
INFO:tensorflow:global_step/sec: 1.9604
I0725 07:52:19.720485 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9604
INFO:tensorflow:loss = 3.0655315, step = 8700 (51.011 sec)
I0725 07:52:19.722162 139696487446400 basic_session_run_hooks.py:260] loss = 3.0655315, step = 8700 (51.011 sec)
INFO:tensorflow:global_step/sec: 1.98225
I0725 07:53:10.168154 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98225
INFO:tensorflow:loss = 4.758828, step = 8800 (50.447 sec)
I0725 07:53:10.169588 139696487446400 basic_session_run_hooks.py:260] loss = 4.758828, step = 8800 (50.447 sec)
INFO:tensorflow:global_step/sec: 1.97282
I0725 07:54:00.856923 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97282
INFO:tensorflow:loss = 3.9258657, step = 8900 (50.688 sec)
I0725 07:54:00.858026 139696487446400 basic_session_run_hooks.py:260] loss = 3.9258657, step = 8900 (50.688 sec)
INFO:tensorflow:global_step/sec: 1.97046
I0725 07:54:51.606427 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97046
INFO:tensorflow:loss = 3.2695491, step = 9000 (50.750 sec)
I0725 07:54:51.608286 139696487446400 basic_session_run_hooks.py:260] loss = 3.2695491, step = 9000 (50.750 sec)
INFO:tensorflow:global_step/sec: 1.98151
I0725 07:55:42.072932 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98151
INFO:tensorflow:loss = 3.1448798, step = 9100 (50.466 sec)
I0725 07:55:42.073934 139696487446400 basic_session_run_hooks.py:260] loss = 3.1448798, step = 9100 (50.466 sec)
INFO:tensorflow:global_step/sec: 1.97826
I0725 07:56:32.622367 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97826
INFO:tensorflow:loss = 2.988894, step = 9200 (50.550 sec)
I0725 07:56:32.623602 139696487446400 basic_session_run_hooks.py:260] loss = 2.988894, step = 9200 (50.550 sec)
INFO:tensorflow:Saving checkpoints for 9209 into training/model.ckpt.
I0725 07:56:36.781165 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 9209 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd314e358>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 07:56:38.337206 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd314e358>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd28aff28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 07:56:38.550384 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd28aff28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 07:56:39.181032 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.780974 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.820449 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.858645 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.897536 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.935960 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 07:56:41.975275 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 07:56:44.141038 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T07:56:44Z
I0725 07:56:44.160999 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T07:56:44Z
INFO:tensorflow:Graph was finalized.
I0725 07:56:44.690370 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 07:56:44.691250: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:56:44.691845: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 07:56:44.691939: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 07:56:44.692041: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 07:56:44.692092: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 07:56:44.692141: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 07:56:44.692189: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 07:56:44.692235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 07:56:44.692281: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 07:56:44.692421: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:56:44.693078: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:56:44.693564: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 07:56:44.693693: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 07:56:44.693722: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 07:56:44.693737: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 07:56:44.693909: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:56:44.694477: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 07:56:44.694992: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-9209
I0725 07:56:44.696304 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-9209
INFO:tensorflow:Running local_init_op.
I0725 07:56:45.667674 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 07:56:45.844377 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 07:56:56.549506 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 07:56:56.550914 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 07:56:56.552428 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.08s).
Accumulating evaluation results...
DONE (t=0.09s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.002
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.002
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.069
INFO:tensorflow:Finished evaluation at 2020-07-25-07:56:57
I0725 07:56:57.830203 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-07:56:57
INFO:tensorflow:Saving dict for global step 9209: DetectionBoxes_Precision/mAP = 1.0873488e-05, DetectionBoxes_Precision/mAP (large) = 0.00011442401, DetectionBoxes_Precision/mAP (medium) = 0.0, DetectionBoxes_Precision/mAP (small) = 0.0, DetectionBoxes_Precision/mAP@.50IOU = 4.480086e-05, DetectionBoxes_Precision/mAP@.75IOU = 8.95612e-06, DetectionBoxes_Recall/AR@1 = 0.00011976048, DetectionBoxes_Recall/AR@10 = 0.0017964072, DetectionBoxes_Recall/AR@100 = 0.0023952096, DetectionBoxes_Recall/AR@100 (large) = 0.06896552, DetectionBoxes_Recall/AR@100 (medium) = 0.0, DetectionBoxes_Recall/AR@100 (small) = 0.0, Loss/classification_loss = 9.405179, Loss/localization_loss = 3.3668585, Loss/regularization_loss = 0.77793413, Loss/total_loss = 13.5499735, global_step = 9209, learning_rate = 0.003, loss = 13.5499735
I0725 07:56:57.830510 139696487446400 estimator.py:2049] Saving dict for global step 9209: DetectionBoxes_Precision/mAP = 1.0873488e-05, DetectionBoxes_Precision/mAP (large) = 0.00011442401, DetectionBoxes_Precision/mAP (medium) = 0.0, DetectionBoxes_Precision/mAP (small) = 0.0, DetectionBoxes_Precision/mAP@.50IOU = 4.480086e-05, DetectionBoxes_Precision/mAP@.75IOU = 8.95612e-06, DetectionBoxes_Recall/AR@1 = 0.00011976048, DetectionBoxes_Recall/AR@10 = 0.0017964072, DetectionBoxes_Recall/AR@100 = 0.0023952096, DetectionBoxes_Recall/AR@100 (large) = 0.06896552, DetectionBoxes_Recall/AR@100 (medium) = 0.0, DetectionBoxes_Recall/AR@100 (small) = 0.0, Loss/classification_loss = 9.405179, Loss/localization_loss = 3.3668585, Loss/regularization_loss = 0.77793413, Loss/total_loss = 13.5499735, global_step = 9209, learning_rate = 0.003, loss = 13.5499735
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 9209: training/model.ckpt-9209
I0725 07:56:57.834752 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 9209: training/model.ckpt-9209
INFO:tensorflow:global_step/sec: 1.3795
I0725 07:57:45.112468 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3795
INFO:tensorflow:loss = 3.3648806, step = 9300 (72.490 sec)
I0725 07:57:45.113803 139696487446400 basic_session_run_hooks.py:260] loss = 3.3648806, step = 9300 (72.490 sec)
INFO:tensorflow:global_step/sec: 1.96825
I0725 07:58:35.918988 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96825
INFO:tensorflow:loss = 4.10993, step = 9400 (50.806 sec)
I0725 07:58:35.920304 139696487446400 basic_session_run_hooks.py:260] loss = 4.10993, step = 9400 (50.806 sec)
INFO:tensorflow:global_step/sec: 1.96722
I0725 07:59:26.752202 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96722
INFO:tensorflow:loss = 3.4890928, step = 9500 (50.834 sec)
I0725 07:59:26.753872 139696487446400 basic_session_run_hooks.py:260] loss = 3.4890928, step = 9500 (50.834 sec)
INFO:tensorflow:global_step/sec: 1.96124
I0725 08:00:17.740360 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96124
INFO:tensorflow:loss = 3.8091683, step = 9600 (50.988 sec)
I0725 08:00:17.741563 139696487446400 basic_session_run_hooks.py:260] loss = 3.8091683, step = 9600 (50.988 sec)
INFO:tensorflow:global_step/sec: 1.97766
I0725 08:01:08.305214 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97766
INFO:tensorflow:loss = 4.071062, step = 9700 (50.565 sec)
I0725 08:01:08.306577 139696487446400 basic_session_run_hooks.py:260] loss = 4.071062, step = 9700 (50.565 sec)
INFO:tensorflow:global_step/sec: 1.98015
I0725 08:01:58.806288 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98015
INFO:tensorflow:loss = 3.6783948, step = 9800 (50.501 sec)
I0725 08:01:58.807433 139696487446400 basic_session_run_hooks.py:260] loss = 3.6783948, step = 9800 (50.501 sec)
INFO:tensorflow:global_step/sec: 1.96276
I0725 08:02:49.755060 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96276
INFO:tensorflow:loss = 3.382586, step = 9900 (50.949 sec)
I0725 08:02:49.756292 139696487446400 basic_session_run_hooks.py:260] loss = 3.382586, step = 9900 (50.949 sec)
INFO:tensorflow:global_step/sec: 1.96117
I0725 08:03:40.744976 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96117
INFO:tensorflow:loss = 3.443975, step = 10000 (50.990 sec)
I0725 08:03:40.746139 139696487446400 basic_session_run_hooks.py:260] loss = 3.443975, step = 10000 (50.990 sec)
INFO:tensorflow:global_step/sec: 1.98129
I0725 08:04:31.217188 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98129
INFO:tensorflow:loss = 3.8036206, step = 10100 (50.472 sec)
I0725 08:04:31.218244 139696487446400 basic_session_run_hooks.py:260] loss = 3.8036206, step = 10100 (50.472 sec)
INFO:tensorflow:global_step/sec: 1.97465
I0725 08:05:21.859148 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97465
INFO:tensorflow:loss = 3.7766535, step = 10200 (50.642 sec)
I0725 08:05:21.860321 139696487446400 basic_session_run_hooks.py:260] loss = 3.7766535, step = 10200 (50.642 sec)
INFO:tensorflow:global_step/sec: 1.96359
I0725 08:06:12.786230 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96359
INFO:tensorflow:loss = 3.2156315, step = 10300 (50.927 sec)
I0725 08:06:12.787379 139696487446400 basic_session_run_hooks.py:260] loss = 3.2156315, step = 10300 (50.927 sec)
INFO:tensorflow:Saving checkpoints for 10349 into training/model.ckpt.
I0725 08:06:36.862243 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 10349 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd5539518>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:06:38.460116 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd5539518>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d2032fa60> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:06:38.662502 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d2032fa60> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:06:39.274368 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:41.855865 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:41.894418 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:41.930232 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:41.967659 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:42.005387 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:06:42.043251 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:06:43.819291 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:06:43Z
I0725 08:06:43.840483 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:06:43Z
INFO:tensorflow:Graph was finalized.
I0725 08:06:44.365580 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:06:44.366348: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:06:44.366980: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:06:44.367105: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:06:44.367156: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:06:44.367212: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:06:44.367270: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:06:44.367354: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:06:44.367403: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:06:44.367452: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:06:44.367584: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:06:44.368245: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:06:44.368786: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:06:44.368855: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:06:44.368879: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:06:44.368894: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:06:44.369108: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:06:44.369665: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:06:44.370165: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-10349
I0725 08:06:44.371290 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-10349
INFO:tensorflow:Running local_init_op.
I0725 08:06:45.305019 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:06:45.444097 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:06:56.000649 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:06:56.002043 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:06:56.005879 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.02s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.014
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.056
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.003
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.005
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.064
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.000
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.017
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.043
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.052
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.020
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.088
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.059
INFO:tensorflow:Finished evaluation at 2020-07-25-08:06:57
I0725 08:06:57.228301 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:06:57
INFO:tensorflow:Saving dict for global step 10349: DetectionBoxes_Precision/mAP = 0.014320001, DetectionBoxes_Precision/mAP (large) = 9.28493e-05, DetectionBoxes_Precision/mAP (medium) = 0.064482875, DetectionBoxes_Precision/mAP (small) = 0.0049496265, DetectionBoxes_Precision/mAP@.50IOU = 0.056416146, DetectionBoxes_Precision/mAP@.75IOU = 0.0029197678, DetectionBoxes_Recall/AR@1 = 0.016646707, DetectionBoxes_Recall/AR@10 = 0.043353293, DetectionBoxes_Recall/AR@100 = 0.051976047, DetectionBoxes_Recall/AR@100 (large) = 0.05862069, DetectionBoxes_Recall/AR@100 (medium) = 0.08832891, DetectionBoxes_Recall/AR@100 (small) = 0.019534884, Loss/classification_loss = 7.9249883, Loss/localization_loss = 2.403447, Loss/regularization_loss = 0.7484589, Loss/total_loss = 11.07689, global_step = 10349, learning_rate = 0.003, loss = 11.07689
I0725 08:06:57.228643 139696487446400 estimator.py:2049] Saving dict for global step 10349: DetectionBoxes_Precision/mAP = 0.014320001, DetectionBoxes_Precision/mAP (large) = 9.28493e-05, DetectionBoxes_Precision/mAP (medium) = 0.064482875, DetectionBoxes_Precision/mAP (small) = 0.0049496265, DetectionBoxes_Precision/mAP@.50IOU = 0.056416146, DetectionBoxes_Precision/mAP@.75IOU = 0.0029197678, DetectionBoxes_Recall/AR@1 = 0.016646707, DetectionBoxes_Recall/AR@10 = 0.043353293, DetectionBoxes_Recall/AR@100 = 0.051976047, DetectionBoxes_Recall/AR@100 (large) = 0.05862069, DetectionBoxes_Recall/AR@100 (medium) = 0.08832891, DetectionBoxes_Recall/AR@100 (small) = 0.019534884, Loss/classification_loss = 7.9249883, Loss/localization_loss = 2.403447, Loss/regularization_loss = 0.7484589, Loss/total_loss = 11.07689, global_step = 10349, learning_rate = 0.003, loss = 11.07689
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 10349: training/model.ckpt-10349
I0725 08:06:57.232388 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 10349: training/model.ckpt-10349
INFO:tensorflow:global_step/sec: 1.41222
I0725 08:07:23.596934 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.41222
INFO:tensorflow:loss = 3.6554098, step = 10400 (70.811 sec)
I0725 08:07:23.598417 139696487446400 basic_session_run_hooks.py:260] loss = 3.6554098, step = 10400 (70.811 sec)
INFO:tensorflow:global_step/sec: 1.974
I0725 08:08:14.255469 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.974
INFO:tensorflow:loss = 3.6832838, step = 10500 (50.659 sec)
I0725 08:08:14.257371 139696487446400 basic_session_run_hooks.py:260] loss = 3.6832838, step = 10500 (50.659 sec)
INFO:tensorflow:global_step/sec: 1.96316
I0725 08:09:05.193747 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96316
INFO:tensorflow:loss = 3.9410377, step = 10600 (50.938 sec)
I0725 08:09:05.195070 139696487446400 basic_session_run_hooks.py:260] loss = 3.9410377, step = 10600 (50.938 sec)
INFO:tensorflow:global_step/sec: 1.98202
I0725 08:09:55.647321 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98202
INFO:tensorflow:loss = 3.4412158, step = 10700 (50.454 sec)
I0725 08:09:55.648707 139696487446400 basic_session_run_hooks.py:260] loss = 3.4412158, step = 10700 (50.454 sec)
INFO:tensorflow:global_step/sec: 1.97566
I0725 08:10:46.263243 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97566
INFO:tensorflow:loss = 3.5880058, step = 10800 (50.616 sec)
I0725 08:10:46.264793 139696487446400 basic_session_run_hooks.py:260] loss = 3.5880058, step = 10800 (50.616 sec)
INFO:tensorflow:global_step/sec: 1.98188
I0725 08:11:36.720413 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98188
INFO:tensorflow:loss = 3.9700665, step = 10900 (50.457 sec)
I0725 08:11:36.721500 139696487446400 basic_session_run_hooks.py:260] loss = 3.9700665, step = 10900 (50.457 sec)
INFO:tensorflow:global_step/sec: 1.98148
I0725 08:12:27.187802 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98148
INFO:tensorflow:loss = 3.6005266, step = 11000 (50.468 sec)
I0725 08:12:27.189334 139696487446400 basic_session_run_hooks.py:260] loss = 3.6005266, step = 11000 (50.468 sec)
INFO:tensorflow:global_step/sec: 1.97675
I0725 08:13:17.776012 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97675
INFO:tensorflow:loss = 3.767958, step = 11100 (50.588 sec)
I0725 08:13:17.777266 139696487446400 basic_session_run_hooks.py:260] loss = 3.767958, step = 11100 (50.588 sec)
INFO:tensorflow:global_step/sec: 1.97448
I0725 08:14:08.422108 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97448
INFO:tensorflow:loss = 3.642569, step = 11200 (50.646 sec)
I0725 08:14:08.423313 139696487446400 basic_session_run_hooks.py:260] loss = 3.642569, step = 11200 (50.646 sec)
INFO:tensorflow:global_step/sec: 1.98527
I0725 08:14:58.793109 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98527
INFO:tensorflow:loss = 2.9490871, step = 11300 (50.371 sec)
I0725 08:14:58.794525 139696487446400 basic_session_run_hooks.py:260] loss = 2.9490871, step = 11300 (50.371 sec)
INFO:tensorflow:global_step/sec: 1.98039
I0725 08:15:49.288138 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98039
INFO:tensorflow:loss = 3.5516114, step = 11400 (50.495 sec)
I0725 08:15:49.289308 139696487446400 basic_session_run_hooks.py:260] loss = 3.5516114, step = 11400 (50.495 sec)
INFO:tensorflow:Saving checkpoints for 11496 into training/model.ckpt.
I0725 08:16:37.239850 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 11496 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd5390358>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:16:38.825991 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd5390358>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd53afd08> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:16:39.030632 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd53afd08> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:16:39.638712 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.583298 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.624343 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.662858 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.701520 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.741238 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:16:42.785677 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:16:44.558166 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:16:44Z
I0725 08:16:44.577937 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:16:44Z
INFO:tensorflow:Graph was finalized.
I0725 08:16:45.091289 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:16:45.092125: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:16:45.092710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:16:45.092798: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:16:45.092874: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:16:45.092923: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:16:45.092971: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:16:45.093024: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:16:45.093075: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:16:45.093126: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:16:45.093275: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:16:45.093855: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:16:45.094381: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:16:45.094535: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:16:45.094575: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:16:45.094589: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:16:45.094768: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:16:45.095391: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:16:45.095891: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-11496
I0725 08:16:45.097279 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-11496
INFO:tensorflow:Running local_init_op.
I0725 08:16:46.046599 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:16:46.186601 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:16:56.867050 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:16:56.868726 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:16:56.871635 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.01s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.210
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.477
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.136
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.081
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.365
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.341
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.105
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.258
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.266
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.125
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.410
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.466
INFO:tensorflow:Finished evaluation at 2020-07-25-08:16:58
I0725 08:16:58.103597 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:16:58
INFO:tensorflow:Saving dict for global step 11496: DetectionBoxes_Precision/mAP = 0.20973198, DetectionBoxes_Precision/mAP (large) = 0.34091455, DetectionBoxes_Precision/mAP (medium) = 0.3649149, DetectionBoxes_Precision/mAP (small) = 0.08061566, DetectionBoxes_Precision/mAP@.50IOU = 0.47707134, DetectionBoxes_Precision/mAP@.75IOU = 0.13628478, DetectionBoxes_Recall/AR@1 = 0.10526946, DetectionBoxes_Recall/AR@10 = 0.25760478, DetectionBoxes_Recall/AR@100 = 0.26550898, DetectionBoxes_Recall/AR@100 (large) = 0.46551725, DetectionBoxes_Recall/AR@100 (medium) = 0.40981433, DetectionBoxes_Recall/AR@100 (small) = 0.12488372, Loss/classification_loss = 5.490633, Loss/localization_loss = 1.2133976, Loss/regularization_loss = 0.7206476, Loss/total_loss = 7.424679, global_step = 11496, learning_rate = 0.003, loss = 7.424679
I0725 08:16:58.103929 139696487446400 estimator.py:2049] Saving dict for global step 11496: DetectionBoxes_Precision/mAP = 0.20973198, DetectionBoxes_Precision/mAP (large) = 0.34091455, DetectionBoxes_Precision/mAP (medium) = 0.3649149, DetectionBoxes_Precision/mAP (small) = 0.08061566, DetectionBoxes_Precision/mAP@.50IOU = 0.47707134, DetectionBoxes_Precision/mAP@.75IOU = 0.13628478, DetectionBoxes_Recall/AR@1 = 0.10526946, DetectionBoxes_Recall/AR@10 = 0.25760478, DetectionBoxes_Recall/AR@100 = 0.26550898, DetectionBoxes_Recall/AR@100 (large) = 0.46551725, DetectionBoxes_Recall/AR@100 (medium) = 0.40981433, DetectionBoxes_Recall/AR@100 (small) = 0.12488372, Loss/classification_loss = 5.490633, Loss/localization_loss = 1.2133976, Loss/regularization_loss = 0.7206476, Loss/total_loss = 7.424679, global_step = 11496, learning_rate = 0.003, loss = 7.424679
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 11496: training/model.ckpt-11496
I0725 08:16:58.109047 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 11496: training/model.ckpt-11496
INFO:tensorflow:global_step/sec: 1.39875
I0725 08:17:00.780408 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.39875
INFO:tensorflow:loss = 3.4053228, step = 11500 (71.492 sec)
I0725 08:17:00.781630 139696487446400 basic_session_run_hooks.py:260] loss = 3.4053228, step = 11500 (71.492 sec)
INFO:tensorflow:global_step/sec: 1.97569
I0725 08:17:51.395725 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97569
INFO:tensorflow:loss = 4.099991, step = 11600 (50.615 sec)
I0725 08:17:51.396990 139696487446400 basic_session_run_hooks.py:260] loss = 4.099991, step = 11600 (50.615 sec)
INFO:tensorflow:global_step/sec: 1.96834
I0725 08:18:42.200026 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96834
INFO:tensorflow:loss = 2.5698175, step = 11700 (50.804 sec)
I0725 08:18:42.201304 139696487446400 basic_session_run_hooks.py:260] loss = 2.5698175, step = 11700 (50.804 sec)
INFO:tensorflow:global_step/sec: 1.96945
I0725 08:19:32.975570 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96945
INFO:tensorflow:loss = 4.393279, step = 11800 (50.776 sec)
I0725 08:19:32.977039 139696487446400 basic_session_run_hooks.py:260] loss = 4.393279, step = 11800 (50.776 sec)
INFO:tensorflow:global_step/sec: 1.97977
I0725 08:20:23.486444 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97977
INFO:tensorflow:loss = 4.2092633, step = 11900 (50.511 sec)
I0725 08:20:23.487723 139696487446400 basic_session_run_hooks.py:260] loss = 4.2092633, step = 11900 (50.511 sec)
INFO:tensorflow:global_step/sec: 1.99157
I0725 08:21:13.698161 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99157
INFO:tensorflow:loss = 3.7189665, step = 12000 (50.212 sec)
I0725 08:21:13.699387 139696487446400 basic_session_run_hooks.py:260] loss = 3.7189665, step = 12000 (50.212 sec)
INFO:tensorflow:global_step/sec: 1.96817
I0725 08:22:04.506714 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96817
INFO:tensorflow:loss = 3.4076355, step = 12100 (50.809 sec)
I0725 08:22:04.508315 139696487446400 basic_session_run_hooks.py:260] loss = 3.4076355, step = 12100 (50.809 sec)
INFO:tensorflow:global_step/sec: 1.98113
I0725 08:22:54.983035 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98113
INFO:tensorflow:loss = 2.8362942, step = 12200 (50.476 sec)
I0725 08:22:54.984437 139696487446400 basic_session_run_hooks.py:260] loss = 2.8362942, step = 12200 (50.476 sec)
INFO:tensorflow:global_step/sec: 1.98834
I0725 08:23:45.276171 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98834
INFO:tensorflow:loss = 3.553363, step = 12300 (50.293 sec)
I0725 08:23:45.277428 139696487446400 basic_session_run_hooks.py:260] loss = 3.553363, step = 12300 (50.293 sec)
INFO:tensorflow:global_step/sec: 1.95982
I0725 08:24:36.301332 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95982
INFO:tensorflow:loss = 3.131826, step = 12400 (51.025 sec)
I0725 08:24:36.302637 139696487446400 basic_session_run_hooks.py:260] loss = 3.131826, step = 12400 (51.025 sec)
INFO:tensorflow:global_step/sec: 1.97326
I0725 08:25:26.978904 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97326
INFO:tensorflow:loss = 2.9793968, step = 12500 (50.677 sec)
I0725 08:25:26.980110 139696487446400 basic_session_run_hooks.py:260] loss = 2.9793968, step = 12500 (50.677 sec)
INFO:tensorflow:global_step/sec: 1.96816
I0725 08:26:17.787554 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96816
INFO:tensorflow:loss = 4.138257, step = 12600 (50.809 sec)
I0725 08:26:17.788940 139696487446400 basic_session_run_hooks.py:260] loss = 4.138257, step = 12600 (50.809 sec)
INFO:tensorflow:Saving checkpoints for 12640 into training/model.ckpt.
I0725 08:26:37.348601 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 12640 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd53ac908>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:26:38.946304 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd53ac908>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd24ad1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:26:39.151781 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd24ad1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:26:39.768455 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.414592 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.454869 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.494290 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.532727 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.570053 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:26:42.608181 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:26:44.672513 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:26:44Z
I0725 08:26:44.692790 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:26:44Z
INFO:tensorflow:Graph was finalized.
I0725 08:26:45.244525 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:26:45.245387: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:26:45.245961: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:26:45.246098: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:26:45.246175: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:26:45.246220: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:26:45.246279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:26:45.246331: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:26:45.246375: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:26:45.246431: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:26:45.246622: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:26:45.247306: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:26:45.247981: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:26:45.248048: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:26:45.248077: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:26:45.248093: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:26:45.248274: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:26:45.248866: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:26:45.249443: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-12640
I0725 08:26:45.250651 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-12640
INFO:tensorflow:Running local_init_op.
I0725 08:26:46.217860 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:26:46.362345 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:26:57.178396 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:26:57.179830 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:26:57.181397 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.01s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.213
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.533
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.132
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.075
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.392
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.692
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.118
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.293
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.304
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.144
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.452
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.741
INFO:tensorflow:Finished evaluation at 2020-07-25-08:26:58
I0725 08:26:58.394794 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:26:58
INFO:tensorflow:Saving dict for global step 12640: DetectionBoxes_Precision/mAP = 0.21328352, DetectionBoxes_Precision/mAP (large) = 0.6922883, DetectionBoxes_Precision/mAP (medium) = 0.39234287, DetectionBoxes_Precision/mAP (small) = 0.07532794, DetectionBoxes_Precision/mAP@.50IOU = 0.5326269, DetectionBoxes_Precision/mAP@.75IOU = 0.13180567, DetectionBoxes_Recall/AR@1 = 0.11760479, DetectionBoxes_Recall/AR@10 = 0.29281437, DetectionBoxes_Recall/AR@100 = 0.30431136, DetectionBoxes_Recall/AR@100 (large) = 0.7413793, DetectionBoxes_Recall/AR@100 (medium) = 0.45225465, DetectionBoxes_Recall/AR@100 (small) = 0.1444186, Loss/classification_loss = 4.268654, Loss/localization_loss = 1.1253685, Loss/regularization_loss = 0.6946733, Loss/total_loss = 6.0886965, global_step = 12640, learning_rate = 0.003, loss = 6.0886965
I0725 08:26:58.395232 139696487446400 estimator.py:2049] Saving dict for global step 12640: DetectionBoxes_Precision/mAP = 0.21328352, DetectionBoxes_Precision/mAP (large) = 0.6922883, DetectionBoxes_Precision/mAP (medium) = 0.39234287, DetectionBoxes_Precision/mAP (small) = 0.07532794, DetectionBoxes_Precision/mAP@.50IOU = 0.5326269, DetectionBoxes_Precision/mAP@.75IOU = 0.13180567, DetectionBoxes_Recall/AR@1 = 0.11760479, DetectionBoxes_Recall/AR@10 = 0.29281437, DetectionBoxes_Recall/AR@100 = 0.30431136, DetectionBoxes_Recall/AR@100 (large) = 0.7413793, DetectionBoxes_Recall/AR@100 (medium) = 0.45225465, DetectionBoxes_Recall/AR@100 (small) = 0.1444186, Loss/classification_loss = 4.268654, Loss/localization_loss = 1.1253685, Loss/regularization_loss = 0.6946733, Loss/total_loss = 6.0886965, global_step = 12640, learning_rate = 0.003, loss = 6.0886965
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 12640: training/model.ckpt-12640
I0725 08:26:58.400598 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 12640: training/model.ckpt-12640
INFO:tensorflow:global_step/sec: 1.39203
I0725 08:27:29.625120 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.39203
INFO:tensorflow:loss = 3.3019667, step = 12700 (71.837 sec)
I0725 08:27:29.626110 139696487446400 basic_session_run_hooks.py:260] loss = 3.3019667, step = 12700 (71.837 sec)
INFO:tensorflow:global_step/sec: 1.97341
I0725 08:28:20.298844 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97341
INFO:tensorflow:loss = 3.026133, step = 12800 (50.674 sec)
I0725 08:28:20.300133 139696487446400 basic_session_run_hooks.py:260] loss = 3.026133, step = 12800 (50.674 sec)
INFO:tensorflow:global_step/sec: 1.96737
I0725 08:29:11.127972 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96737
INFO:tensorflow:loss = 3.9587808, step = 12900 (50.829 sec)
I0725 08:29:11.129192 139696487446400 basic_session_run_hooks.py:260] loss = 3.9587808, step = 12900 (50.829 sec)
INFO:tensorflow:global_step/sec: 1.99162
I0725 08:30:01.338320 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99162
INFO:tensorflow:loss = 3.5034847, step = 13000 (50.211 sec)
I0725 08:30:01.339892 139696487446400 basic_session_run_hooks.py:260] loss = 3.5034847, step = 13000 (50.211 sec)
INFO:tensorflow:global_step/sec: 1.98129
I0725 08:30:51.810447 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98129
INFO:tensorflow:loss = 3.4510825, step = 13100 (50.472 sec)
I0725 08:30:51.811737 139696487446400 basic_session_run_hooks.py:260] loss = 3.4510825, step = 13100 (50.472 sec)
INFO:tensorflow:global_step/sec: 1.96894
I0725 08:31:42.599212 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96894
INFO:tensorflow:loss = 3.7794056, step = 13200 (50.789 sec)
I0725 08:31:42.600398 139696487446400 basic_session_run_hooks.py:260] loss = 3.7794056, step = 13200 (50.789 sec)
INFO:tensorflow:global_step/sec: 1.98821
I0725 08:32:32.895886 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98821
INFO:tensorflow:loss = 3.8143294, step = 13300 (50.297 sec)
I0725 08:32:32.896919 139696487446400 basic_session_run_hooks.py:260] loss = 3.8143294, step = 13300 (50.297 sec)
INFO:tensorflow:global_step/sec: 1.97646
I0725 08:33:23.491286 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97646
INFO:tensorflow:loss = 2.8231642, step = 13400 (50.596 sec)
I0725 08:33:23.492653 139696487446400 basic_session_run_hooks.py:260] loss = 2.8231642, step = 13400 (50.596 sec)
INFO:tensorflow:global_step/sec: 1.98619
I0725 08:34:13.838969 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98619
INFO:tensorflow:loss = 3.4119024, step = 13500 (50.348 sec)
I0725 08:34:13.840367 139696487446400 basic_session_run_hooks.py:260] loss = 3.4119024, step = 13500 (50.348 sec)
INFO:tensorflow:global_step/sec: 1.98125
I0725 08:35:04.312153 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98125
INFO:tensorflow:loss = 3.1621523, step = 13600 (50.473 sec)
I0725 08:35:04.313338 139696487446400 basic_session_run_hooks.py:260] loss = 3.1621523, step = 13600 (50.473 sec)
INFO:tensorflow:global_step/sec: 1.97427
I0725 08:35:54.963913 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97427
INFO:tensorflow:loss = 3.1934237, step = 13700 (50.652 sec)
I0725 08:35:54.965382 139696487446400 basic_session_run_hooks.py:260] loss = 3.1934237, step = 13700 (50.652 sec)
INFO:tensorflow:Saving checkpoints for 13786 into training/model.ckpt.
I0725 08:36:37.692198 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 13786 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd2f86cf8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:36:39.304100 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd2f86cf8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d36c15378> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:36:39.513771 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d36c15378> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:36:40.121553 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.741014 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.785408 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.825084 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.863171 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.901188 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:36:42.939239 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:36:44.745492 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:36:44Z
I0725 08:36:44.765386 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:36:44Z
INFO:tensorflow:Graph was finalized.
I0725 08:36:45.292980 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:36:45.293883: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:36:45.294388: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:36:45.294486: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:36:45.294552: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:36:45.294607: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:36:45.294672: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:36:45.294727: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:36:45.294789: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:36:45.294871: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:36:45.295028: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:36:45.295829: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:36:45.296321: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:36:45.296443: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:36:45.296471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:36:45.296486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:36:45.296694: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:36:45.297294: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:36:45.297938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-13786
I0725 08:36:45.299064 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-13786
INFO:tensorflow:Running local_init_op.
I0725 08:36:46.272159 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:36:46.404738 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:36:57.407877 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:36:57.409000 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:36:57.411151 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.02s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.287
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.621
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.207
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.132
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.481
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.704
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.137
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.369
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.375
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.203
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.541
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.734
INFO:tensorflow:Finished evaluation at 2020-07-25-08:36:58
I0725 08:36:58.671502 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:36:58
INFO:tensorflow:Saving dict for global step 13786: DetectionBoxes_Precision/mAP = 0.2867538, DetectionBoxes_Precision/mAP (large) = 0.7040996, DetectionBoxes_Precision/mAP (medium) = 0.4806484, DetectionBoxes_Precision/mAP (small) = 0.13191035, DetectionBoxes_Precision/mAP@.50IOU = 0.6212685, DetectionBoxes_Precision/mAP@.75IOU = 0.20705302, DetectionBoxes_Recall/AR@1 = 0.13700598, DetectionBoxes_Recall/AR@10 = 0.36862275, DetectionBoxes_Recall/AR@100 = 0.37461078, DetectionBoxes_Recall/AR@100 (large) = 0.73448277, DetectionBoxes_Recall/AR@100 (medium) = 0.54111403, DetectionBoxes_Recall/AR@100 (small) = 0.20348836, Loss/classification_loss = 3.334464, Loss/localization_loss = 0.8354427, Loss/regularization_loss = 0.6704933, Loss/total_loss = 4.8404007, global_step = 13786, learning_rate = 0.003, loss = 4.8404007
I0725 08:36:58.671935 139696487446400 estimator.py:2049] Saving dict for global step 13786: DetectionBoxes_Precision/mAP = 0.2867538, DetectionBoxes_Precision/mAP (large) = 0.7040996, DetectionBoxes_Precision/mAP (medium) = 0.4806484, DetectionBoxes_Precision/mAP (small) = 0.13191035, DetectionBoxes_Precision/mAP@.50IOU = 0.6212685, DetectionBoxes_Precision/mAP@.75IOU = 0.20705302, DetectionBoxes_Recall/AR@1 = 0.13700598, DetectionBoxes_Recall/AR@10 = 0.36862275, DetectionBoxes_Recall/AR@100 = 0.37461078, DetectionBoxes_Recall/AR@100 (large) = 0.73448277, DetectionBoxes_Recall/AR@100 (medium) = 0.54111403, DetectionBoxes_Recall/AR@100 (small) = 0.20348836, Loss/classification_loss = 3.334464, Loss/localization_loss = 0.8354427, Loss/regularization_loss = 0.6704933, Loss/total_loss = 4.8404007, global_step = 13786, learning_rate = 0.003, loss = 4.8404007
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 13786: training/model.ckpt-13786
I0725 08:36:58.676126 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 13786: training/model.ckpt-13786
INFO:tensorflow:global_step/sec: 1.40297
I0725 08:37:06.241106 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.40297
INFO:tensorflow:loss = 3.0350795, step = 13800 (71.277 sec)
I0725 08:37:06.242254 139696487446400 basic_session_run_hooks.py:260] loss = 3.0350795, step = 13800 (71.277 sec)
INFO:tensorflow:global_step/sec: 1.97839
I0725 08:37:56.787263 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97839
INFO:tensorflow:loss = 3.38277, step = 13900 (50.546 sec)
I0725 08:37:56.788508 139696487446400 basic_session_run_hooks.py:260] loss = 3.38277, step = 13900 (50.546 sec)
INFO:tensorflow:global_step/sec: 1.97272
I0725 08:38:47.478594 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97272
INFO:tensorflow:loss = 2.575201, step = 14000 (50.692 sec)
I0725 08:38:47.480309 139696487446400 basic_session_run_hooks.py:260] loss = 2.575201, step = 14000 (50.692 sec)
INFO:tensorflow:global_step/sec: 1.96738
I0725 08:39:38.307709 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96738
INFO:tensorflow:loss = 3.813052, step = 14100 (50.829 sec)
I0725 08:39:38.308963 139696487446400 basic_session_run_hooks.py:260] loss = 3.813052, step = 14100 (50.829 sec)
INFO:tensorflow:global_step/sec: 1.97943
I0725 08:40:28.827220 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97943
INFO:tensorflow:loss = 2.8242822, step = 14200 (50.520 sec)
I0725 08:40:28.828550 139696487446400 basic_session_run_hooks.py:260] loss = 2.8242822, step = 14200 (50.520 sec)
INFO:tensorflow:global_step/sec: 1.97312
I0725 08:41:19.508445 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97312
INFO:tensorflow:loss = 2.2910135, step = 14300 (50.681 sec)
I0725 08:41:19.510039 139696487446400 basic_session_run_hooks.py:260] loss = 2.2910135, step = 14300 (50.681 sec)
INFO:tensorflow:global_step/sec: 1.96762
I0725 08:42:10.331142 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96762
INFO:tensorflow:loss = 2.989688, step = 14400 (50.822 sec)
I0725 08:42:10.332187 139696487446400 basic_session_run_hooks.py:260] loss = 2.989688, step = 14400 (50.822 sec)
INFO:tensorflow:global_step/sec: 1.9792
I0725 08:43:00.856696 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9792
INFO:tensorflow:loss = 2.6251335, step = 14500 (50.526 sec)
I0725 08:43:00.858279 139696487446400 basic_session_run_hooks.py:260] loss = 2.6251335, step = 14500 (50.526 sec)
INFO:tensorflow:global_step/sec: 1.97087
I0725 08:43:51.595909 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97087
INFO:tensorflow:loss = 2.6739025, step = 14600 (50.739 sec)
I0725 08:43:51.597371 139696487446400 basic_session_run_hooks.py:260] loss = 2.6739025, step = 14600 (50.739 sec)
INFO:tensorflow:global_step/sec: 1.97492
I0725 08:44:42.230721 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97492
INFO:tensorflow:loss = 3.3162308, step = 14700 (50.634 sec)
I0725 08:44:42.231869 139696487446400 basic_session_run_hooks.py:260] loss = 3.3162308, step = 14700 (50.634 sec)
INFO:tensorflow:global_step/sec: 1.98293
I0725 08:45:32.661087 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98293
INFO:tensorflow:loss = 3.4933922, step = 14800 (50.431 sec)
I0725 08:45:32.662413 139696487446400 basic_session_run_hooks.py:260] loss = 3.4933922, step = 14800 (50.431 sec)
INFO:tensorflow:global_step/sec: 1.97996
I0725 08:46:23.167315 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97996
INFO:tensorflow:loss = 2.9256735, step = 14900 (50.506 sec)
I0725 08:46:23.168886 139696487446400 basic_session_run_hooks.py:260] loss = 2.9256735, step = 14900 (50.506 sec)
INFO:tensorflow:Saving checkpoints for 14930 into training/model.ckpt.
I0725 08:46:37.755500 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 14930 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37c2550>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:46:39.440068 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37c2550>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3864840> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:46:39.656314 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3864840> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:46:40.297001 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.457171 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.512755 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.556104 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.595052 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.632264 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:46:43.670765 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:46:45.502515 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:46:45Z
I0725 08:46:45.524216 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:46:45Z
INFO:tensorflow:Graph was finalized.
I0725 08:46:46.062384 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:46:46.063174: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:46:46.063753: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:46:46.063956: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:46:46.064023: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:46:46.064072: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:46:46.064122: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:46:46.064178: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:46:46.064235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:46:46.064284: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:46:46.064428: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:46:46.065103: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:46:46.065604: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:46:46.065674: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:46:46.065700: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:46:46.065714: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:46:46.065888: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:46:46.066468: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:46:46.066979: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-14930
I0725 08:46:46.068372 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-14930
INFO:tensorflow:Running local_init_op.
I0725 08:46:47.076396 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:46:47.234472 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:46:58.535971 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:46:58.538602 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:46:58.541937 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.08s).
Accumulating evaluation results...
DONE (t=0.07s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.291
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.632
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.216
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.131
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.480
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.779
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.142
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.373
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.381
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.204
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.549
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.810
INFO:tensorflow:Finished evaluation at 2020-07-25-08:46:59
I0725 08:46:59.798205 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:46:59
INFO:tensorflow:Saving dict for global step 14930: DetectionBoxes_Precision/mAP = 0.29089344, DetectionBoxes_Precision/mAP (large) = 0.77898055, DetectionBoxes_Precision/mAP (medium) = 0.48009244, DetectionBoxes_Precision/mAP (small) = 0.13125005, DetectionBoxes_Precision/mAP@.50IOU = 0.63240975, DetectionBoxes_Precision/mAP@.75IOU = 0.21633169, DetectionBoxes_Recall/AR@1 = 0.14179641, DetectionBoxes_Recall/AR@10 = 0.37293413, DetectionBoxes_Recall/AR@100 = 0.38107786, DetectionBoxes_Recall/AR@100 (large) = 0.8103448, DetectionBoxes_Recall/AR@100 (medium) = 0.5485411, DetectionBoxes_Recall/AR@100 (small) = 0.2044186, Loss/classification_loss = 3.2384772, Loss/localization_loss = 0.8045491, Loss/regularization_loss = 0.6475769, Loss/total_loss = 4.6906033, global_step = 14930, learning_rate = 0.003, loss = 4.6906033
I0725 08:46:59.798598 139696487446400 estimator.py:2049] Saving dict for global step 14930: DetectionBoxes_Precision/mAP = 0.29089344, DetectionBoxes_Precision/mAP (large) = 0.77898055, DetectionBoxes_Precision/mAP (medium) = 0.48009244, DetectionBoxes_Precision/mAP (small) = 0.13125005, DetectionBoxes_Precision/mAP@.50IOU = 0.63240975, DetectionBoxes_Precision/mAP@.75IOU = 0.21633169, DetectionBoxes_Recall/AR@1 = 0.14179641, DetectionBoxes_Recall/AR@10 = 0.37293413, DetectionBoxes_Recall/AR@100 = 0.38107786, DetectionBoxes_Recall/AR@100 (large) = 0.8103448, DetectionBoxes_Recall/AR@100 (medium) = 0.5485411, DetectionBoxes_Recall/AR@100 (small) = 0.2044186, Loss/classification_loss = 3.2384772, Loss/localization_loss = 0.8045491, Loss/regularization_loss = 0.6475769, Loss/total_loss = 4.6906033, global_step = 14930, learning_rate = 0.003, loss = 4.6906033
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 14930: training/model.ckpt-14930
I0725 08:46:59.804921 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 14930: training/model.ckpt-14930
INFO:tensorflow:global_step/sec: 1.36991
I0725 08:47:36.164600 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.36991
INFO:tensorflow:loss = 3.1900516, step = 15000 (72.997 sec)
I0725 08:47:36.165965 139696487446400 basic_session_run_hooks.py:260] loss = 3.1900516, step = 15000 (72.997 sec)
INFO:tensorflow:global_step/sec: 1.97112
I0725 08:48:26.897254 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97112
INFO:tensorflow:loss = 3.23808, step = 15100 (50.732 sec)
I0725 08:48:26.898464 139696487446400 basic_session_run_hooks.py:260] loss = 3.23808, step = 15100 (50.732 sec)
INFO:tensorflow:global_step/sec: 1.98459
I0725 08:49:17.285542 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98459
INFO:tensorflow:loss = 4.38983, step = 15200 (50.388 sec)
I0725 08:49:17.286835 139696487446400 basic_session_run_hooks.py:260] loss = 4.38983, step = 15200 (50.388 sec)
INFO:tensorflow:global_step/sec: 1.97156
I0725 08:50:08.007021 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97156
INFO:tensorflow:loss = 2.9315007, step = 15300 (50.721 sec)
I0725 08:50:08.008164 139696487446400 basic_session_run_hooks.py:260] loss = 2.9315007, step = 15300 (50.721 sec)
INFO:tensorflow:global_step/sec: 1.95982
I0725 08:50:59.032006 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95982
INFO:tensorflow:loss = 3.3584862, step = 15400 (51.025 sec)
I0725 08:50:59.033289 139696487446400 basic_session_run_hooks.py:260] loss = 3.3584862, step = 15400 (51.025 sec)
INFO:tensorflow:global_step/sec: 1.97729
I0725 08:51:49.606300 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97729
INFO:tensorflow:loss = 2.6464262, step = 15500 (50.574 sec)
I0725 08:51:49.607415 139696487446400 basic_session_run_hooks.py:260] loss = 2.6464262, step = 15500 (50.574 sec)
INFO:tensorflow:global_step/sec: 1.96795
I0725 08:52:40.420741 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96795
INFO:tensorflow:loss = 3.044556, step = 15600 (50.815 sec)
I0725 08:52:40.422194 139696487446400 basic_session_run_hooks.py:260] loss = 3.044556, step = 15600 (50.815 sec)
INFO:tensorflow:global_step/sec: 1.97795
I0725 08:53:30.978114 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97795
INFO:tensorflow:loss = 2.9149418, step = 15700 (50.557 sec)
I0725 08:53:30.979417 139696487446400 basic_session_run_hooks.py:260] loss = 2.9149418, step = 15700 (50.557 sec)
INFO:tensorflow:global_step/sec: 1.94509
I0725 08:54:22.389554 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94509
INFO:tensorflow:loss = 2.9456499, step = 15800 (51.411 sec)
I0725 08:54:22.390912 139696487446400 basic_session_run_hooks.py:260] loss = 2.9456499, step = 15800 (51.411 sec)
INFO:tensorflow:global_step/sec: 1.9763
I0725 08:55:12.989234 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9763
INFO:tensorflow:loss = 3.4436312, step = 15900 (50.600 sec)
I0725 08:55:12.990510 139696487446400 basic_session_run_hooks.py:260] loss = 3.4436312, step = 15900 (50.600 sec)
INFO:tensorflow:global_step/sec: 1.97285
I0725 08:56:03.677363 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97285
INFO:tensorflow:loss = 3.23234, step = 16000 (50.688 sec)
I0725 08:56:03.678785 139696487446400 basic_session_run_hooks.py:260] loss = 3.23234, step = 16000 (50.688 sec)
INFO:tensorflow:Saving checkpoints for 16068 into training/model.ckpt.
I0725 08:56:37.857637 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 16068 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd25b0b38>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 08:56:39.560706 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd25b0b38>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d203b71e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 08:56:39.789710 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d203b71e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 08:56:40.433782 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.180517 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.219156 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.257497 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.295331 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.332418 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 08:56:43.369629 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 08:56:45.473150 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T08:56:45Z
I0725 08:56:45.494388 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T08:56:45Z
INFO:tensorflow:Graph was finalized.
I0725 08:56:46.049040 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 08:56:46.049828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:56:46.050408: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 08:56:46.050512: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 08:56:46.050575: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 08:56:46.050627: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 08:56:46.050689: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 08:56:46.050744: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 08:56:46.050790: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 08:56:46.050871: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 08:56:46.051016: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:56:46.051694: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:56:46.052212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 08:56:46.052352: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 08:56:46.052392: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 08:56:46.052408: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 08:56:46.052607: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:56:46.053288: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 08:56:46.053888: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-16068
I0725 08:56:46.055116 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-16068
INFO:tensorflow:Running local_init_op.
I0725 08:56:47.120081 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 08:56:47.267433 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 08:56:58.421110 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 08:56:58.422334 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 08:56:58.425506 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.04s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.329
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.667
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.279
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.155
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.500
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.730
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.148
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.394
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.398
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.218
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.574
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.786
INFO:tensorflow:Finished evaluation at 2020-07-25-08:56:59
I0725 08:56:59.667910 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-08:56:59
INFO:tensorflow:Saving dict for global step 16068: DetectionBoxes_Precision/mAP = 0.328634, DetectionBoxes_Precision/mAP (large) = 0.7299076, DetectionBoxes_Precision/mAP (medium) = 0.500416, DetectionBoxes_Precision/mAP (small) = 0.15520251, DetectionBoxes_Precision/mAP@.50IOU = 0.66723543, DetectionBoxes_Precision/mAP@.75IOU = 0.27897575, DetectionBoxes_Recall/AR@1 = 0.14838323, DetectionBoxes_Recall/AR@10 = 0.39389223, DetectionBoxes_Recall/AR@100 = 0.39832336, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.57427055, DetectionBoxes_Recall/AR@100 (small) = 0.21837209, Loss/classification_loss = 3.0613554, Loss/localization_loss = 0.70374906, Loss/regularization_loss = 0.6263734, Loss/total_loss = 4.39148, global_step = 16068, learning_rate = 0.003, loss = 4.39148
I0725 08:56:59.668259 139696487446400 estimator.py:2049] Saving dict for global step 16068: DetectionBoxes_Precision/mAP = 0.328634, DetectionBoxes_Precision/mAP (large) = 0.7299076, DetectionBoxes_Precision/mAP (medium) = 0.500416, DetectionBoxes_Precision/mAP (small) = 0.15520251, DetectionBoxes_Precision/mAP@.50IOU = 0.66723543, DetectionBoxes_Precision/mAP@.75IOU = 0.27897575, DetectionBoxes_Recall/AR@1 = 0.14838323, DetectionBoxes_Recall/AR@10 = 0.39389223, DetectionBoxes_Recall/AR@100 = 0.39832336, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.57427055, DetectionBoxes_Recall/AR@100 (small) = 0.21837209, Loss/classification_loss = 3.0613554, Loss/localization_loss = 0.70374906, Loss/regularization_loss = 0.6263734, Loss/total_loss = 4.39148, global_step = 16068, learning_rate = 0.003, loss = 4.39148
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 16068: training/model.ckpt-16068
I0725 08:56:59.672497 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 16068: training/model.ckpt-16068
INFO:tensorflow:global_step/sec: 1.37168
I0725 08:57:16.580870 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.37168
INFO:tensorflow:loss = 3.162843, step = 16100 (72.903 sec)
I0725 08:57:16.581908 139696487446400 basic_session_run_hooks.py:260] loss = 3.162843, step = 16100 (72.903 sec)
INFO:tensorflow:global_step/sec: 1.95588
I0725 08:58:07.708675 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95588
INFO:tensorflow:loss = 3.4203238, step = 16200 (51.128 sec)
I0725 08:58:07.709768 139696487446400 basic_session_run_hooks.py:260] loss = 3.4203238, step = 16200 (51.128 sec)
INFO:tensorflow:global_step/sec: 1.96504
I0725 08:58:58.598196 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96504
INFO:tensorflow:loss = 3.2746933, step = 16300 (50.890 sec)
I0725 08:58:58.599532 139696487446400 basic_session_run_hooks.py:260] loss = 3.2746933, step = 16300 (50.890 sec)
INFO:tensorflow:global_step/sec: 1.96955
I0725 08:59:49.371282 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96955
INFO:tensorflow:loss = 2.9624443, step = 16400 (50.773 sec)
I0725 08:59:49.372437 139696487446400 basic_session_run_hooks.py:260] loss = 2.9624443, step = 16400 (50.773 sec)
INFO:tensorflow:global_step/sec: 1.96596
I0725 09:00:40.236896 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96596
INFO:tensorflow:loss = 3.0282977, step = 16500 (50.866 sec)
I0725 09:00:40.238128 139696487446400 basic_session_run_hooks.py:260] loss = 3.0282977, step = 16500 (50.866 sec)
INFO:tensorflow:global_step/sec: 1.97249
I0725 09:01:30.934192 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97249
INFO:tensorflow:loss = 2.6466362, step = 16600 (50.697 sec)
I0725 09:01:30.935317 139696487446400 basic_session_run_hooks.py:260] loss = 2.6466362, step = 16600 (50.697 sec)
INFO:tensorflow:global_step/sec: 1.94967
I0725 09:02:22.224944 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94967
INFO:tensorflow:loss = 2.9594936, step = 16700 (51.291 sec)
I0725 09:02:22.226109 139696487446400 basic_session_run_hooks.py:260] loss = 2.9594936, step = 16700 (51.291 sec)
INFO:tensorflow:global_step/sec: 1.96271
I0725 09:03:13.174758 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96271
INFO:tensorflow:loss = 3.3606415, step = 16800 (50.950 sec)
I0725 09:03:13.176168 139696487446400 basic_session_run_hooks.py:260] loss = 3.3606415, step = 16800 (50.950 sec)
INFO:tensorflow:global_step/sec: 1.96782
I0725 09:04:03.992460 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96782
INFO:tensorflow:loss = 2.7779155, step = 16900 (50.818 sec)
I0725 09:04:03.993902 139696487446400 basic_session_run_hooks.py:260] loss = 2.7779155, step = 16900 (50.818 sec)
INFO:tensorflow:global_step/sec: 1.98408
I0725 09:04:54.393686 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98408
INFO:tensorflow:loss = 3.4757376, step = 17000 (50.401 sec)
I0725 09:04:54.395311 139696487446400 basic_session_run_hooks.py:260] loss = 3.4757376, step = 17000 (50.401 sec)
INFO:tensorflow:global_step/sec: 1.98501
I0725 09:05:44.771293 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98501
INFO:tensorflow:loss = 3.1273973, step = 17100 (50.377 sec)
I0725 09:05:44.772527 139696487446400 basic_session_run_hooks.py:260] loss = 3.1273973, step = 17100 (50.377 sec)
INFO:tensorflow:global_step/sec: 1.97983
I0725 09:06:35.280885 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97983
INFO:tensorflow:loss = 2.6742067, step = 17200 (50.514 sec)
I0725 09:06:35.286376 139696487446400 basic_session_run_hooks.py:260] loss = 2.6742067, step = 17200 (50.514 sec)
INFO:tensorflow:Saving checkpoints for 17206 into training/model.ckpt.
I0725 09:06:37.920323 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 17206 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3993588>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:06:39.550408 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3993588>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2cd4a60> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:06:39.779169 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2cd4a60> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:06:40.384505 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.022312 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.060713 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.097437 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.140527 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.177762 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:06:43.215585 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:06:45.013527 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:06:45Z
I0725 09:06:45.033982 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:06:45Z
INFO:tensorflow:Graph was finalized.
I0725 09:06:45.547152 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:06:45.547886: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:06:45.548456: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:06:45.548578: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:06:45.548625: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:06:45.548693: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:06:45.548740: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:06:45.548791: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:06:45.548849: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:06:45.548898: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:06:45.549043: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:06:45.549584: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:06:45.550165: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:06:45.550221: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:06:45.550255: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:06:45.550276: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:06:45.550433: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:06:45.551176: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:06:45.551745: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-17206
I0725 09:06:45.553188 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-17206
INFO:tensorflow:Running local_init_op.
I0725 09:06:46.504963 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:06:46.642536 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:06:57.777300 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:06:57.778791 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:06:57.780605 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.02s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.318
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.650
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.267
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.139
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.508
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.714
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.387
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.392
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.209
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.570
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.779
INFO:tensorflow:Finished evaluation at 2020-07-25-09:06:59
I0725 09:06:59.014695 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:06:59
INFO:tensorflow:Saving dict for global step 17206: DetectionBoxes_Precision/mAP = 0.31761563, DetectionBoxes_Precision/mAP (large) = 0.71384907, DetectionBoxes_Precision/mAP (medium) = 0.50818646, DetectionBoxes_Precision/mAP (small) = 0.13869064, DetectionBoxes_Precision/mAP@.50IOU = 0.65028465, DetectionBoxes_Precision/mAP@.75IOU = 0.2670963, DetectionBoxes_Recall/AR@1 = 0.14658682, DetectionBoxes_Recall/AR@10 = 0.38658682, DetectionBoxes_Recall/AR@100 = 0.3920958, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.5700265, DetectionBoxes_Recall/AR@100 (small) = 0.20930232, Loss/classification_loss = 3.1167479, Loss/localization_loss = 0.7578185, Loss/regularization_loss = 0.6067432, Loss/total_loss = 4.4813094, global_step = 17206, learning_rate = 0.003, loss = 4.4813094
I0725 09:06:59.015033 139696487446400 estimator.py:2049] Saving dict for global step 17206: DetectionBoxes_Precision/mAP = 0.31761563, DetectionBoxes_Precision/mAP (large) = 0.71384907, DetectionBoxes_Precision/mAP (medium) = 0.50818646, DetectionBoxes_Precision/mAP (small) = 0.13869064, DetectionBoxes_Precision/mAP@.50IOU = 0.65028465, DetectionBoxes_Precision/mAP@.75IOU = 0.2670963, DetectionBoxes_Recall/AR@1 = 0.14658682, DetectionBoxes_Recall/AR@10 = 0.38658682, DetectionBoxes_Recall/AR@100 = 0.3920958, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.5700265, DetectionBoxes_Recall/AR@100 (small) = 0.20930232, Loss/classification_loss = 3.1167479, Loss/localization_loss = 0.7578185, Loss/regularization_loss = 0.6067432, Loss/total_loss = 4.4813094, global_step = 17206, learning_rate = 0.003, loss = 4.4813094
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 17206: training/model.ckpt-17206
I0725 09:06:59.018990 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 17206: training/model.ckpt-17206
INFO:tensorflow:global_step/sec: 1.38698
I0725 09:07:47.379847 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38698
INFO:tensorflow:loss = 2.9021072, step = 17300 (72.095 sec)
I0725 09:07:47.381062 139696487446400 basic_session_run_hooks.py:260] loss = 2.9021072, step = 17300 (72.095 sec)
INFO:tensorflow:global_step/sec: 1.97729
I0725 09:08:37.954074 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97729
INFO:tensorflow:loss = 3.5714548, step = 17400 (50.574 sec)
I0725 09:08:37.955311 139696487446400 basic_session_run_hooks.py:260] loss = 3.5714548, step = 17400 (50.574 sec)
INFO:tensorflow:global_step/sec: 1.97635
I0725 09:09:28.552494 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97635
INFO:tensorflow:loss = 2.3075411, step = 17500 (50.598 sec)
I0725 09:09:28.553791 139696487446400 basic_session_run_hooks.py:260] loss = 2.3075411, step = 17500 (50.598 sec)
INFO:tensorflow:global_step/sec: 1.96346
I0725 09:10:19.482840 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96346
INFO:tensorflow:loss = 3.9488108, step = 17600 (50.931 sec)
I0725 09:10:19.484305 139696487446400 basic_session_run_hooks.py:260] loss = 3.9488108, step = 17600 (50.931 sec)
INFO:tensorflow:global_step/sec: 1.96739
I0725 09:11:10.311389 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96739
INFO:tensorflow:loss = 2.7973924, step = 17700 (50.828 sec)
I0725 09:11:10.312611 139696487446400 basic_session_run_hooks.py:260] loss = 2.7973924, step = 17700 (50.828 sec)
INFO:tensorflow:global_step/sec: 1.9613
I0725 09:12:01.297986 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9613
INFO:tensorflow:loss = 3.0281754, step = 17800 (50.987 sec)
I0725 09:12:01.299196 139696487446400 basic_session_run_hooks.py:260] loss = 3.0281754, step = 17800 (50.987 sec)
INFO:tensorflow:global_step/sec: 1.96563
I0725 09:12:52.172274 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96563
INFO:tensorflow:loss = 2.9989033, step = 17900 (50.874 sec)
I0725 09:12:52.173660 139696487446400 basic_session_run_hooks.py:260] loss = 2.9989033, step = 17900 (50.874 sec)
INFO:tensorflow:global_step/sec: 1.96851
I0725 09:13:42.972042 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96851
INFO:tensorflow:loss = 2.3699076, step = 18000 (50.800 sec)
I0725 09:13:42.973361 139696487446400 basic_session_run_hooks.py:260] loss = 2.3699076, step = 18000 (50.800 sec)
INFO:tensorflow:global_step/sec: 1.97678
I0725 09:14:33.559454 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97678
INFO:tensorflow:loss = 2.8316355, step = 18100 (50.587 sec)
I0725 09:14:33.560857 139696487446400 basic_session_run_hooks.py:260] loss = 2.8316355, step = 18100 (50.587 sec)
INFO:tensorflow:global_step/sec: 1.97477
I0725 09:15:24.198238 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97477
INFO:tensorflow:loss = 2.4330723, step = 18200 (50.639 sec)
I0725 09:15:24.199670 139696487446400 basic_session_run_hooks.py:260] loss = 2.4330723, step = 18200 (50.639 sec)
INFO:tensorflow:global_step/sec: 1.98612
I0725 09:16:14.547669 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98612
INFO:tensorflow:loss = 2.7088082, step = 18300 (50.349 sec)
I0725 09:16:14.548877 139696487446400 basic_session_run_hooks.py:260] loss = 2.7088082, step = 18300 (50.349 sec)
INFO:tensorflow:Saving checkpoints for 18347 into training/model.ckpt.
I0725 09:16:37.992809 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 18347 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd39c44e0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:16:39.663931 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd39c44e0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a3c488> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:16:39.874397 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a3c488> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:16:40.507237 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.642296 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.682070 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.729493 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.768915 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.806409 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:16:43.844920 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:16:45.670840 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:16:45Z
I0725 09:16:45.691264 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:16:45Z
INFO:tensorflow:Graph was finalized.
I0725 09:16:46.222569 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:16:46.223442: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:16:46.224139: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:16:46.224250: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:16:46.224315: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:16:46.224394: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:16:46.224450: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:16:46.224523: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:16:46.224577: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:16:46.224627: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:16:46.224792: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:16:46.225360: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:16:46.225887: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:16:46.226025: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:16:46.226065: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:16:46.226105: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:16:46.226346: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:16:46.227015: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:16:46.227522: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-18347
I0725 09:16:46.229092 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-18347
INFO:tensorflow:Running local_init_op.
I0725 09:16:47.252220 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:16:47.404808 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:16:58.981638 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:16:58.983083 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:16:58.987231 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.08s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.314
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.631
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.267
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.161
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.501
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.736
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.148
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.406
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.414
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.243
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.581
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.776
INFO:tensorflow:Finished evaluation at 2020-07-25-09:17:00
I0725 09:17:00.267105 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:17:00
INFO:tensorflow:Saving dict for global step 18347: DetectionBoxes_Precision/mAP = 0.3143673, DetectionBoxes_Precision/mAP (large) = 0.7356819, DetectionBoxes_Precision/mAP (medium) = 0.50107014, DetectionBoxes_Precision/mAP (small) = 0.1608978, DetectionBoxes_Precision/mAP@.50IOU = 0.6311705, DetectionBoxes_Precision/mAP@.75IOU = 0.26681083, DetectionBoxes_Recall/AR@1 = 0.14790419, DetectionBoxes_Recall/AR@10 = 0.4063473, DetectionBoxes_Recall/AR@100 = 0.41437125, DetectionBoxes_Recall/AR@100 (large) = 0.7758621, DetectionBoxes_Recall/AR@100 (medium) = 0.58143234, DetectionBoxes_Recall/AR@100 (small) = 0.24348837, Loss/classification_loss = 3.010199, Loss/localization_loss = 0.6999114, Loss/regularization_loss = 0.5881179, Loss/total_loss = 4.298228, global_step = 18347, learning_rate = 0.003, loss = 4.298228
I0725 09:17:00.267556 139696487446400 estimator.py:2049] Saving dict for global step 18347: DetectionBoxes_Precision/mAP = 0.3143673, DetectionBoxes_Precision/mAP (large) = 0.7356819, DetectionBoxes_Precision/mAP (medium) = 0.50107014, DetectionBoxes_Precision/mAP (small) = 0.1608978, DetectionBoxes_Precision/mAP@.50IOU = 0.6311705, DetectionBoxes_Precision/mAP@.75IOU = 0.26681083, DetectionBoxes_Recall/AR@1 = 0.14790419, DetectionBoxes_Recall/AR@10 = 0.4063473, DetectionBoxes_Recall/AR@100 = 0.41437125, DetectionBoxes_Recall/AR@100 (large) = 0.7758621, DetectionBoxes_Recall/AR@100 (medium) = 0.58143234, DetectionBoxes_Recall/AR@100 (small) = 0.24348837, Loss/classification_loss = 3.010199, Loss/localization_loss = 0.6999114, Loss/regularization_loss = 0.5881179, Loss/total_loss = 4.298228, global_step = 18347, learning_rate = 0.003, loss = 4.298228
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 18347: training/model.ckpt-18347
I0725 09:17:00.271930 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 18347: training/model.ckpt-18347
INFO:tensorflow:global_step/sec: 1.36626
I0725 09:17:27.740283 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.36626
INFO:tensorflow:loss = 3.3928561, step = 18400 (73.193 sec)
I0725 09:17:27.741459 139696487446400 basic_session_run_hooks.py:260] loss = 3.3928561, step = 18400 (73.193 sec)
INFO:tensorflow:global_step/sec: 1.96878
I0725 09:18:18.533146 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96878
INFO:tensorflow:loss = 2.889301, step = 18500 (50.793 sec)
I0725 09:18:18.534403 139696487446400 basic_session_run_hooks.py:260] loss = 2.889301, step = 18500 (50.793 sec)
INFO:tensorflow:global_step/sec: 1.96065
I0725 09:19:09.536527 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96065
INFO:tensorflow:loss = 2.0716028, step = 18600 (51.003 sec)
I0725 09:19:09.537886 139696487446400 basic_session_run_hooks.py:260] loss = 2.0716028, step = 18600 (51.003 sec)
INFO:tensorflow:global_step/sec: 1.94713
I0725 09:20:00.894175 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94713
INFO:tensorflow:loss = 2.5879664, step = 18700 (51.358 sec)
I0725 09:20:00.895639 139696487446400 basic_session_run_hooks.py:260] loss = 2.5879664, step = 18700 (51.358 sec)
INFO:tensorflow:global_step/sec: 1.96793
I0725 09:20:51.709060 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96793
INFO:tensorflow:loss = 3.0046818, step = 18800 (50.815 sec)
I0725 09:20:51.710185 139696487446400 basic_session_run_hooks.py:260] loss = 3.0046818, step = 18800 (50.815 sec)
INFO:tensorflow:global_step/sec: 1.96815
I0725 09:21:42.518070 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96815
INFO:tensorflow:loss = 2.1900148, step = 18900 (50.809 sec)
I0725 09:21:42.519289 139696487446400 basic_session_run_hooks.py:260] loss = 2.1900148, step = 18900 (50.809 sec)
INFO:tensorflow:global_step/sec: 1.97817
I0725 09:22:33.069988 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97817
INFO:tensorflow:loss = 2.5920005, step = 19000 (50.552 sec)
I0725 09:22:33.071264 139696487446400 basic_session_run_hooks.py:260] loss = 2.5920005, step = 19000 (50.552 sec)
INFO:tensorflow:global_step/sec: 1.97235
I0725 09:23:23.771031 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97235
INFO:tensorflow:loss = 2.927903, step = 19100 (50.701 sec)
I0725 09:23:23.772392 139696487446400 basic_session_run_hooks.py:260] loss = 2.927903, step = 19100 (50.701 sec)
INFO:tensorflow:global_step/sec: 1.9659
I0725 09:24:14.638232 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9659
INFO:tensorflow:loss = 2.6686745, step = 19200 (50.867 sec)
I0725 09:24:14.639619 139696487446400 basic_session_run_hooks.py:260] loss = 2.6686745, step = 19200 (50.867 sec)
INFO:tensorflow:global_step/sec: 1.98205
I0725 09:25:05.091080 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98205
INFO:tensorflow:loss = 2.273131, step = 19300 (50.453 sec)
I0725 09:25:05.092387 139696487446400 basic_session_run_hooks.py:260] loss = 2.273131, step = 19300 (50.453 sec)
INFO:tensorflow:global_step/sec: 1.96774
I0725 09:25:55.910875 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96774
INFO:tensorflow:loss = 3.3658667, step = 19400 (50.820 sec)
I0725 09:25:55.912199 139696487446400 basic_session_run_hooks.py:260] loss = 3.3658667, step = 19400 (50.820 sec)
INFO:tensorflow:Saving checkpoints for 19484 into training/model.ckpt.
I0725 09:26:38.179554 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 19484 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3b49470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:26:39.884960 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3b49470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2bac1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:26:40.118872 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2bac1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:26:40.784052 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.559764 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.605324 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.643904 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.682902 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.722862 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:26:43.761565 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:26:45.949658 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:26:45Z
I0725 09:26:45.971449 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:26:45Z
INFO:tensorflow:Graph was finalized.
I0725 09:26:46.523277 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:26:46.524161: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:26:46.524776: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:26:46.524966: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:26:46.525033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:26:46.525090: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:26:46.525154: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:26:46.525217: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:26:46.525268: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:26:46.525316: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:26:46.525469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:26:46.526170: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:26:46.526667: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:26:46.526785: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:26:46.526862: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:26:46.526889: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:26:46.527085: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:26:46.527711: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:26:46.528257: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-19484
I0725 09:26:46.529643 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-19484
INFO:tensorflow:Running local_init_op.
I0725 09:26:47.538710 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:26:47.704544 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:26:59.162317 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:26:59.163385 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:26:59.166743 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.06s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.309
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.659
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.251
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.155
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.482
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.707
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.143
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.392
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.401
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.238
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.560
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.766
INFO:tensorflow:Finished evaluation at 2020-07-25-09:27:00
I0725 09:27:00.432785 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:27:00
INFO:tensorflow:Saving dict for global step 19484: DetectionBoxes_Precision/mAP = 0.30920246, DetectionBoxes_Precision/mAP (large) = 0.7070073, DetectionBoxes_Precision/mAP (medium) = 0.48196068, DetectionBoxes_Precision/mAP (small) = 0.15478536, DetectionBoxes_Precision/mAP@.50IOU = 0.65874827, DetectionBoxes_Precision/mAP@.75IOU = 0.25101253, DetectionBoxes_Recall/AR@1 = 0.14323354, DetectionBoxes_Recall/AR@10 = 0.39233533, DetectionBoxes_Recall/AR@100 = 0.40143713, DetectionBoxes_Recall/AR@100 (large) = 0.76551723, DetectionBoxes_Recall/AR@100 (medium) = 0.5596817, DetectionBoxes_Recall/AR@100 (small) = 0.23790698, Loss/classification_loss = 3.06738, Loss/localization_loss = 0.7553652, Loss/regularization_loss = 0.57084095, Loss/total_loss = 4.3935866, global_step = 19484, learning_rate = 0.003, loss = 4.3935866
I0725 09:27:00.433179 139696487446400 estimator.py:2049] Saving dict for global step 19484: DetectionBoxes_Precision/mAP = 0.30920246, DetectionBoxes_Precision/mAP (large) = 0.7070073, DetectionBoxes_Precision/mAP (medium) = 0.48196068, DetectionBoxes_Precision/mAP (small) = 0.15478536, DetectionBoxes_Precision/mAP@.50IOU = 0.65874827, DetectionBoxes_Precision/mAP@.75IOU = 0.25101253, DetectionBoxes_Recall/AR@1 = 0.14323354, DetectionBoxes_Recall/AR@10 = 0.39233533, DetectionBoxes_Recall/AR@100 = 0.40143713, DetectionBoxes_Recall/AR@100 (large) = 0.76551723, DetectionBoxes_Recall/AR@100 (medium) = 0.5596817, DetectionBoxes_Recall/AR@100 (small) = 0.23790698, Loss/classification_loss = 3.06738, Loss/localization_loss = 0.7553652, Loss/regularization_loss = 0.57084095, Loss/total_loss = 4.3935866, global_step = 19484, learning_rate = 0.003, loss = 4.3935866
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 19484: training/model.ckpt-19484
I0725 09:27:00.437007 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 19484: training/model.ckpt-19484
INFO:tensorflow:global_step/sec: 1.3653
I0725 09:27:09.154654 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3653
INFO:tensorflow:loss = 1.9586692, step = 19500 (73.244 sec)
I0725 09:27:09.156114 139696487446400 basic_session_run_hooks.py:260] loss = 1.9586692, step = 19500 (73.244 sec)
INFO:tensorflow:global_step/sec: 1.97697
I0725 09:27:59.737094 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97697
INFO:tensorflow:loss = 2.62501, step = 19600 (50.582 sec)
I0725 09:27:59.738386 139696487446400 basic_session_run_hooks.py:260] loss = 2.62501, step = 19600 (50.582 sec)
INFO:tensorflow:global_step/sec: 1.95725
I0725 09:28:50.829216 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95725
INFO:tensorflow:loss = 3.3585162, step = 19700 (51.092 sec)
I0725 09:28:50.830617 139696487446400 basic_session_run_hooks.py:260] loss = 3.3585162, step = 19700 (51.092 sec)
INFO:tensorflow:global_step/sec: 1.96202
I0725 09:29:41.797065 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96202
INFO:tensorflow:loss = 2.9095945, step = 19800 (50.968 sec)
I0725 09:29:41.798757 139696487446400 basic_session_run_hooks.py:260] loss = 2.9095945, step = 19800 (50.968 sec)
INFO:tensorflow:global_step/sec: 1.96884
I0725 09:30:32.588226 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96884
INFO:tensorflow:loss = 3.1392076, step = 19900 (50.791 sec)
I0725 09:30:32.589366 139696487446400 basic_session_run_hooks.py:260] loss = 3.1392076, step = 19900 (50.791 sec)
INFO:tensorflow:global_step/sec: 1.96763
I0725 09:31:23.410694 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96763
INFO:tensorflow:loss = 2.0142367, step = 20000 (50.823 sec)
I0725 09:31:23.411955 139696487446400 basic_session_run_hooks.py:260] loss = 2.0142367, step = 20000 (50.823 sec)
INFO:tensorflow:global_step/sec: 1.96303
I0725 09:32:14.352444 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96303
INFO:tensorflow:loss = 2.9518154, step = 20100 (50.943 sec)
I0725 09:32:14.354489 139696487446400 basic_session_run_hooks.py:260] loss = 2.9518154, step = 20100 (50.943 sec)
INFO:tensorflow:global_step/sec: 1.95875
I0725 09:33:05.405476 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95875
INFO:tensorflow:loss = 2.658029, step = 20200 (51.052 sec)
I0725 09:33:05.406862 139696487446400 basic_session_run_hooks.py:260] loss = 2.658029, step = 20200 (51.052 sec)
INFO:tensorflow:global_step/sec: 1.96584
I0725 09:33:56.274345 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96584
INFO:tensorflow:loss = 3.3373153, step = 20300 (50.869 sec)
I0725 09:33:56.275543 139696487446400 basic_session_run_hooks.py:260] loss = 3.3373153, step = 20300 (50.869 sec)
INFO:tensorflow:global_step/sec: 1.9713
I0725 09:34:47.002433 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9713
INFO:tensorflow:loss = 2.8487496, step = 20400 (50.728 sec)
I0725 09:34:47.003965 139696487446400 basic_session_run_hooks.py:260] loss = 2.8487496, step = 20400 (50.728 sec)
INFO:tensorflow:global_step/sec: 1.97379
I0725 09:35:37.666438 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97379
INFO:tensorflow:loss = 2.946377, step = 20500 (50.664 sec)
I0725 09:35:37.667705 139696487446400 basic_session_run_hooks.py:260] loss = 2.946377, step = 20500 (50.664 sec)
INFO:tensorflow:global_step/sec: 1.95383
I0725 09:36:28.847895 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95383
INFO:tensorflow:loss = 2.1929715, step = 20600 (51.181 sec)
I0725 09:36:28.849126 139696487446400 basic_session_run_hooks.py:260] loss = 2.1929715, step = 20600 (51.181 sec)
INFO:tensorflow:Saving checkpoints for 20620 into training/model.ckpt.
I0725 09:36:38.320190 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 20620 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0ccfaae630>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:36:39.987558 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0ccfaae630>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2f8b510> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:36:40.224017 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2f8b510> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:36:40.870055 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.633917 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.673792 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.713127 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.752102 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.790638 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:36:43.849937 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:36:45.714401 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:36:45Z
I0725 09:36:45.736398 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:36:45Z
INFO:tensorflow:Graph was finalized.
I0725 09:36:46.283445 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:36:46.284317: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:36:46.284984: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:36:46.285095: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:36:46.285164: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:36:46.285226: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:36:46.285286: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:36:46.285341: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:36:46.285393: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:36:46.285447: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:36:46.285593: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:36:46.286227: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:36:46.286766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:36:46.286945: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:36:46.286977: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:36:46.286995: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:36:46.287171: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:36:46.287828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:36:46.288362: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-20620
I0725 09:36:46.289620 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-20620
INFO:tensorflow:Running local_init_op.
I0725 09:36:47.255530 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:36:47.385598 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:36:58.723549 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:36:58.725202 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:36:58.729042 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.07s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.303
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.661
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.221
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.141
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.487
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.712
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.139
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.379
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.387
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.210
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.561
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.769
INFO:tensorflow:Finished evaluation at 2020-07-25-09:37:00
I0725 09:37:00.019988 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:37:00
INFO:tensorflow:Saving dict for global step 20620: DetectionBoxes_Precision/mAP = 0.3028521, DetectionBoxes_Precision/mAP (large) = 0.71234244, DetectionBoxes_Precision/mAP (medium) = 0.48711744, DetectionBoxes_Precision/mAP (small) = 0.14143932, DetectionBoxes_Precision/mAP@.50IOU = 0.66069174, DetectionBoxes_Precision/mAP@.75IOU = 0.22113132, DetectionBoxes_Recall/AR@1 = 0.13892215, DetectionBoxes_Recall/AR@10 = 0.3794012, DetectionBoxes_Recall/AR@100 = 0.38730538, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5607427, DetectionBoxes_Recall/AR@100 (small) = 0.21, Loss/classification_loss = 3.2379022, Loss/localization_loss = 0.7877202, Loss/regularization_loss = 0.5546117, Loss/total_loss = 4.5802355, global_step = 20620, learning_rate = 0.003, loss = 4.5802355
I0725 09:37:00.020325 139696487446400 estimator.py:2049] Saving dict for global step 20620: DetectionBoxes_Precision/mAP = 0.3028521, DetectionBoxes_Precision/mAP (large) = 0.71234244, DetectionBoxes_Precision/mAP (medium) = 0.48711744, DetectionBoxes_Precision/mAP (small) = 0.14143932, DetectionBoxes_Precision/mAP@.50IOU = 0.66069174, DetectionBoxes_Precision/mAP@.75IOU = 0.22113132, DetectionBoxes_Recall/AR@1 = 0.13892215, DetectionBoxes_Recall/AR@10 = 0.3794012, DetectionBoxes_Recall/AR@100 = 0.38730538, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5607427, DetectionBoxes_Recall/AR@100 (small) = 0.21, Loss/classification_loss = 3.2379022, Loss/localization_loss = 0.7877202, Loss/regularization_loss = 0.5546117, Loss/total_loss = 4.5802355, global_step = 20620, learning_rate = 0.003, loss = 4.5802355
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 20620: training/model.ckpt-20620
I0725 09:37:00.024436 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 20620: training/model.ckpt-20620
INFO:tensorflow:global_step/sec: 1.3836
I0725 09:37:41.122927 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3836
INFO:tensorflow:loss = 2.8632932, step = 20700 (72.275 sec)
I0725 09:37:41.124128 139696487446400 basic_session_run_hooks.py:260] loss = 2.8632932, step = 20700 (72.275 sec)
INFO:tensorflow:global_step/sec: 1.97562
I0725 09:38:31.740017 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97562
INFO:tensorflow:loss = 2.2368693, step = 20800 (50.617 sec)
I0725 09:38:31.741298 139696487446400 basic_session_run_hooks.py:260] loss = 2.2368693, step = 20800 (50.617 sec)
INFO:tensorflow:global_step/sec: 1.97121
I0725 09:39:22.470195 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97121
INFO:tensorflow:loss = 2.9097075, step = 20900 (50.730 sec)
I0725 09:39:22.471389 139696487446400 basic_session_run_hooks.py:260] loss = 2.9097075, step = 20900 (50.730 sec)
INFO:tensorflow:global_step/sec: 1.95998
I0725 09:40:13.491266 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95998
INFO:tensorflow:loss = 3.244255, step = 21000 (51.021 sec)
I0725 09:40:13.492477 139696487446400 basic_session_run_hooks.py:260] loss = 3.244255, step = 21000 (51.021 sec)
INFO:tensorflow:global_step/sec: 1.96867
I0725 09:41:04.287087 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96867
INFO:tensorflow:loss = 2.9508, step = 21100 (50.796 sec)
I0725 09:41:04.288270 139696487446400 basic_session_run_hooks.py:260] loss = 2.9508, step = 21100 (50.796 sec)
INFO:tensorflow:global_step/sec: 1.98175
I0725 09:41:54.747515 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98175
INFO:tensorflow:loss = 3.2138565, step = 21200 (50.461 sec)
I0725 09:41:54.749018 139696487446400 basic_session_run_hooks.py:260] loss = 3.2138565, step = 21200 (50.461 sec)
INFO:tensorflow:global_step/sec: 1.97791
I0725 09:42:45.305831 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97791
INFO:tensorflow:loss = 2.6045282, step = 21300 (50.558 sec)
I0725 09:42:45.307231 139696487446400 basic_session_run_hooks.py:260] loss = 2.6045282, step = 21300 (50.558 sec)
INFO:tensorflow:global_step/sec: 1.96903
I0725 09:43:36.092405 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96903
INFO:tensorflow:loss = 2.523848, step = 21400 (50.789 sec)
I0725 09:43:36.096262 139696487446400 basic_session_run_hooks.py:260] loss = 2.523848, step = 21400 (50.789 sec)
INFO:tensorflow:global_step/sec: 1.96639
I0725 09:44:26.947111 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96639
INFO:tensorflow:loss = 3.4475117, step = 21500 (50.852 sec)
I0725 09:44:26.948513 139696487446400 basic_session_run_hooks.py:260] loss = 3.4475117, step = 21500 (50.852 sec)
INFO:tensorflow:global_step/sec: 1.95761
I0725 09:45:18.029762 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95761
INFO:tensorflow:loss = 2.6728163, step = 21600 (51.083 sec)
I0725 09:45:18.031171 139696487446400 basic_session_run_hooks.py:260] loss = 2.6728163, step = 21600 (51.083 sec)
INFO:tensorflow:global_step/sec: 1.96628
I0725 09:46:08.887199 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96628
INFO:tensorflow:loss = 2.9062905, step = 21700 (50.857 sec)
I0725 09:46:08.888401 139696487446400 basic_session_run_hooks.py:260] loss = 2.9062905, step = 21700 (50.857 sec)
INFO:tensorflow:Saving checkpoints for 21759 into training/model.ckpt.
I0725 09:46:38.553094 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 21759 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37ef438>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:46:40.224302 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37ef438>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd37ed0d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:46:40.442453 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd37ed0d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:46:41.103917 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.278242 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.318684 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.358026 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.397530 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.436255 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:46:44.477517 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:46:46.333069 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:46:46Z
I0725 09:46:46.354138 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:46:46Z
INFO:tensorflow:Graph was finalized.
I0725 09:46:46.899419 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:46:46.900257: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:46:46.900944: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:46:46.901126: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:46:46.901182: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:46:46.901293: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:46:46.901419: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:46:46.901487: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:46:46.901539: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:46:46.901590: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:46:46.901773: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:46:46.902423: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:46:46.903017: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:46:46.903093: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:46:46.903119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:46:46.903136: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:46:46.903297: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:46:46.903933: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:46:46.904452: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-21759
I0725 09:46:46.905899 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-21759
INFO:tensorflow:Running local_init_op.
I0725 09:46:47.946372 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:46:48.100127 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:46:59.520800 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:46:59.522318 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:46:59.525445 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.08s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.327
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.669
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.288
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.153
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.501
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.737
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.387
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.396
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.220
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.568
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.779
INFO:tensorflow:Finished evaluation at 2020-07-25-09:47:00
I0725 09:47:00.810301 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:47:00
INFO:tensorflow:Saving dict for global step 21759: DetectionBoxes_Precision/mAP = 0.32666004, DetectionBoxes_Precision/mAP (large) = 0.73701286, DetectionBoxes_Precision/mAP (medium) = 0.50124747, DetectionBoxes_Precision/mAP (small) = 0.153053, DetectionBoxes_Precision/mAP@.50IOU = 0.6693139, DetectionBoxes_Precision/mAP@.75IOU = 0.28838205, DetectionBoxes_Recall/AR@1 = 0.14670658, DetectionBoxes_Recall/AR@10 = 0.38730538, DetectionBoxes_Recall/AR@100 = 0.3962874, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.568435, DetectionBoxes_Recall/AR@100 (small) = 0.22023256, Loss/classification_loss = 3.1864347, Loss/localization_loss = 0.7469111, Loss/regularization_loss = 0.5393393, Loss/total_loss = 4.4726853, global_step = 21759, learning_rate = 0.003, loss = 4.4726853
I0725 09:47:00.810591 139696487446400 estimator.py:2049] Saving dict for global step 21759: DetectionBoxes_Precision/mAP = 0.32666004, DetectionBoxes_Precision/mAP (large) = 0.73701286, DetectionBoxes_Precision/mAP (medium) = 0.50124747, DetectionBoxes_Precision/mAP (small) = 0.153053, DetectionBoxes_Precision/mAP@.50IOU = 0.6693139, DetectionBoxes_Precision/mAP@.75IOU = 0.28838205, DetectionBoxes_Recall/AR@1 = 0.14670658, DetectionBoxes_Recall/AR@10 = 0.38730538, DetectionBoxes_Recall/AR@100 = 0.3962874, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.568435, DetectionBoxes_Recall/AR@100 (small) = 0.22023256, Loss/classification_loss = 3.1864347, Loss/localization_loss = 0.7469111, Loss/regularization_loss = 0.5393393, Loss/total_loss = 4.4726853, global_step = 21759, learning_rate = 0.003, loss = 4.4726853
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 21759: training/model.ckpt-21759
I0725 09:47:00.814974 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 21759: training/model.ckpt-21759
INFO:tensorflow:global_step/sec: 1.36056
I0725 09:47:22.386169 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.36056
INFO:tensorflow:loss = 2.6063142, step = 21800 (73.499 sec)
I0725 09:47:22.387471 139696487446400 basic_session_run_hooks.py:260] loss = 2.6063142, step = 21800 (73.499 sec)
INFO:tensorflow:global_step/sec: 1.9695
I0725 09:48:13.160478 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9695
INFO:tensorflow:loss = 3.4765167, step = 21900 (50.774 sec)
I0725 09:48:13.161593 139696487446400 basic_session_run_hooks.py:260] loss = 3.4765167, step = 21900 (50.774 sec)
INFO:tensorflow:global_step/sec: 1.9687
I0725 09:49:03.955469 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9687
INFO:tensorflow:loss = 2.8910956, step = 22000 (50.795 sec)
I0725 09:49:03.956557 139696487446400 basic_session_run_hooks.py:260] loss = 2.8910956, step = 22000 (50.795 sec)
INFO:tensorflow:global_step/sec: 1.95803
I0725 09:49:55.027125 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95803
INFO:tensorflow:loss = 3.086115, step = 22100 (51.072 sec)
I0725 09:49:55.028536 139696487446400 basic_session_run_hooks.py:260] loss = 3.086115, step = 22100 (51.072 sec)
INFO:tensorflow:global_step/sec: 1.97335
I0725 09:50:45.702216 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97335
INFO:tensorflow:loss = 2.0359352, step = 22200 (50.675 sec)
I0725 09:50:45.703434 139696487446400 basic_session_run_hooks.py:260] loss = 2.0359352, step = 22200 (50.675 sec)
INFO:tensorflow:global_step/sec: 1.95765
I0725 09:51:36.783974 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95765
INFO:tensorflow:loss = 2.4997797, step = 22300 (51.082 sec)
I0725 09:51:36.785164 139696487446400 basic_session_run_hooks.py:260] loss = 2.4997797, step = 22300 (51.082 sec)
INFO:tensorflow:global_step/sec: 1.95782
I0725 09:52:27.861140 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95782
INFO:tensorflow:loss = 3.2006035, step = 22400 (51.081 sec)
I0725 09:52:27.866252 139696487446400 basic_session_run_hooks.py:260] loss = 3.2006035, step = 22400 (51.081 sec)
INFO:tensorflow:global_step/sec: 1.96707
I0725 09:53:18.698167 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96707
INFO:tensorflow:loss = 2.6613069, step = 22500 (50.833 sec)
I0725 09:53:18.699475 139696487446400 basic_session_run_hooks.py:260] loss = 2.6613069, step = 22500 (50.833 sec)
INFO:tensorflow:global_step/sec: 1.96409
I0725 09:54:09.612454 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96409
INFO:tensorflow:loss = 2.9623322, step = 22600 (50.914 sec)
I0725 09:54:09.613749 139696487446400 basic_session_run_hooks.py:260] loss = 2.9623322, step = 22600 (50.914 sec)
INFO:tensorflow:global_step/sec: 1.96547
I0725 09:55:00.490845 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96547
INFO:tensorflow:loss = 3.0483034, step = 22700 (50.878 sec)
I0725 09:55:00.491943 139696487446400 basic_session_run_hooks.py:260] loss = 3.0483034, step = 22700 (50.878 sec)
INFO:tensorflow:global_step/sec: 1.96107
I0725 09:55:51.483403 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96107
INFO:tensorflow:loss = 2.6617918, step = 22800 (50.993 sec)
I0725 09:55:51.484521 139696487446400 basic_session_run_hooks.py:260] loss = 2.6617918, step = 22800 (50.993 sec)
INFO:tensorflow:Saving checkpoints for 22895 into training/model.ckpt.
I0725 09:56:38.770637 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 22895 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37c5470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 09:56:40.433773 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37c5470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2ac41e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 09:56:40.657198 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2ac41e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 09:56:41.299527 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:43.945774 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:43.984096 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:44.023544 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:44.061294 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:44.098959 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 09:56:44.138345 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 09:56:46.001686 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T09:56:46Z
I0725 09:56:46.023404 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T09:56:46Z
INFO:tensorflow:Graph was finalized.
I0725 09:56:46.557113 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 09:56:46.558018: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:56:46.558554: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 09:56:46.558642: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 09:56:46.558704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 09:56:46.558762: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 09:56:46.558839: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 09:56:46.558928: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 09:56:46.558992: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 09:56:46.559041: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 09:56:46.559222: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:56:46.559869: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:56:46.560372: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 09:56:46.560495: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 09:56:46.560523: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 09:56:46.560538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 09:56:46.560692: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:56:46.561283: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 09:56:46.561844: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-22895
I0725 09:56:46.563058 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-22895
INFO:tensorflow:Running local_init_op.
I0725 09:56:47.511053 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 09:56:47.646937 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 09:56:59.230572 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 09:56:59.232245 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 09:56:59.234396 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.03s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.329
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.660
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.292
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.168
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.501
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.737
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.151
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.402
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.407
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.230
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.581
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.776
INFO:tensorflow:Finished evaluation at 2020-07-25-09:57:00
I0725 09:57:00.481673 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-09:57:00
INFO:tensorflow:Saving dict for global step 22895: DetectionBoxes_Precision/mAP = 0.32897112, DetectionBoxes_Precision/mAP (large) = 0.7372985, DetectionBoxes_Precision/mAP (medium) = 0.5009085, DetectionBoxes_Precision/mAP (small) = 0.1677982, DetectionBoxes_Precision/mAP@.50IOU = 0.6597556, DetectionBoxes_Precision/mAP@.75IOU = 0.2924133, DetectionBoxes_Recall/AR@1 = 0.151497, DetectionBoxes_Recall/AR@10 = 0.40239522, DetectionBoxes_Recall/AR@100 = 0.40694612, DetectionBoxes_Recall/AR@100 (large) = 0.7758621, DetectionBoxes_Recall/AR@100 (medium) = 0.5806366, DetectionBoxes_Recall/AR@100 (small) = 0.22976744, Loss/classification_loss = 3.0534222, Loss/localization_loss = 0.6877169, Loss/regularization_loss = 0.52526045, Loss/total_loss = 4.2664003, global_step = 22895, learning_rate = 0.003, loss = 4.2664003
I0725 09:57:00.482304 139696487446400 estimator.py:2049] Saving dict for global step 22895: DetectionBoxes_Precision/mAP = 0.32897112, DetectionBoxes_Precision/mAP (large) = 0.7372985, DetectionBoxes_Precision/mAP (medium) = 0.5009085, DetectionBoxes_Precision/mAP (small) = 0.1677982, DetectionBoxes_Precision/mAP@.50IOU = 0.6597556, DetectionBoxes_Precision/mAP@.75IOU = 0.2924133, DetectionBoxes_Recall/AR@1 = 0.151497, DetectionBoxes_Recall/AR@10 = 0.40239522, DetectionBoxes_Recall/AR@100 = 0.40694612, DetectionBoxes_Recall/AR@100 (large) = 0.7758621, DetectionBoxes_Recall/AR@100 (medium) = 0.5806366, DetectionBoxes_Recall/AR@100 (small) = 0.22976744, Loss/classification_loss = 3.0534222, Loss/localization_loss = 0.6877169, Loss/regularization_loss = 0.52526045, Loss/total_loss = 4.2664003, global_step = 22895, learning_rate = 0.003, loss = 4.2664003
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 22895: training/model.ckpt-22895
I0725 09:57:00.487496 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 22895: training/model.ckpt-22895
INFO:tensorflow:global_step/sec: 1.38478
I0725 09:57:03.697290 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38478
INFO:tensorflow:loss = 2.5953593, step = 22900 (72.214 sec)
I0725 09:57:03.698593 139696487446400 basic_session_run_hooks.py:260] loss = 2.5953593, step = 22900 (72.214 sec)
INFO:tensorflow:global_step/sec: 1.96857
I0725 09:57:54.495604 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96857
INFO:tensorflow:loss = 3.0243063, step = 23000 (50.799 sec)
I0725 09:57:54.497114 139696487446400 basic_session_run_hooks.py:260] loss = 3.0243063, step = 23000 (50.799 sec)
INFO:tensorflow:global_step/sec: 1.9646
I0725 09:58:45.396460 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9646
INFO:tensorflow:loss = 2.3642354, step = 23100 (50.901 sec)
I0725 09:58:45.397639 139696487446400 basic_session_run_hooks.py:260] loss = 2.3642354, step = 23100 (50.901 sec)
INFO:tensorflow:global_step/sec: 1.97072
I0725 09:59:36.139336 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97072
INFO:tensorflow:loss = 2.779873, step = 23200 (50.743 sec)
I0725 09:59:36.140765 139696487446400 basic_session_run_hooks.py:260] loss = 2.779873, step = 23200 (50.743 sec)
INFO:tensorflow:global_step/sec: 1.9591
I0725 10:00:27.183095 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9591
INFO:tensorflow:loss = 3.6995108, step = 23300 (51.044 sec)
I0725 10:00:27.184323 139696487446400 basic_session_run_hooks.py:260] loss = 3.6995108, step = 23300 (51.044 sec)
INFO:tensorflow:global_step/sec: 1.97465
I0725 10:01:17.825086 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97465
INFO:tensorflow:loss = 2.4238875, step = 23400 (50.642 sec)
I0725 10:01:17.826363 139696487446400 basic_session_run_hooks.py:260] loss = 2.4238875, step = 23400 (50.642 sec)
INFO:tensorflow:global_step/sec: 1.96491
I0725 10:02:08.718134 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96491
INFO:tensorflow:loss = 2.8302484, step = 23500 (50.893 sec)
I0725 10:02:08.719459 139696487446400 basic_session_run_hooks.py:260] loss = 2.8302484, step = 23500 (50.893 sec)
INFO:tensorflow:global_step/sec: 1.95334
I0725 10:02:59.912501 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95334
INFO:tensorflow:loss = 2.5224571, step = 23600 (51.194 sec)
I0725 10:02:59.913707 139696487446400 basic_session_run_hooks.py:260] loss = 2.5224571, step = 23600 (51.194 sec)
INFO:tensorflow:global_step/sec: 1.95843
I0725 10:03:50.973703 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95843
INFO:tensorflow:loss = 2.5151656, step = 23700 (51.061 sec)
I0725 10:03:50.974888 139696487446400 basic_session_run_hooks.py:260] loss = 2.5151656, step = 23700 (51.061 sec)
INFO:tensorflow:global_step/sec: 1.97574
I0725 10:04:41.587612 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97574
INFO:tensorflow:loss = 2.023508, step = 23800 (50.614 sec)
I0725 10:04:41.588793 139696487446400 basic_session_run_hooks.py:260] loss = 2.023508, step = 23800 (50.614 sec)
INFO:tensorflow:global_step/sec: 1.99133
I0725 10:05:31.805377 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99133
INFO:tensorflow:loss = 2.5003583, step = 23900 (50.218 sec)
I0725 10:05:31.806543 139696487446400 basic_session_run_hooks.py:260] loss = 2.5003583, step = 23900 (50.218 sec)
INFO:tensorflow:global_step/sec: 1.98081
I0725 10:06:22.289927 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98081
INFO:tensorflow:loss = 2.9446461, step = 24000 (50.485 sec)
I0725 10:06:22.291066 139696487446400 basic_session_run_hooks.py:260] loss = 2.9446461, step = 24000 (50.485 sec)
INFO:tensorflow:Saving checkpoints for 24034 into training/model.ckpt.
I0725 10:06:38.962044 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 24034 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a05198>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:06:40.603307 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a05198>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2ac4b70> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:06:40.826327 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2ac4b70> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:06:41.448872 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.082293 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.121492 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.157946 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.197021 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.235328 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:06:44.271996 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:06:46.059513 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:06:46Z
I0725 10:06:46.082686 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:06:46Z
INFO:tensorflow:Graph was finalized.
I0725 10:06:46.628270 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:06:46.629130: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:06:46.629690: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:06:46.629890: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:06:46.629946: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:06:46.630041: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:06:46.630132: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:06:46.630181: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:06:46.630228: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:06:46.630298: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:06:46.630440: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:06:46.631086: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:06:46.631566: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:06:46.631636: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:06:46.631661: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:06:46.631677: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:06:46.631883: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:06:46.632469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:06:46.633005: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-24034
I0725 10:06:46.634136 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-24034
INFO:tensorflow:Running local_init_op.
I0725 10:06:47.572126 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:06:47.744631 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:06:58.973547 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:06:58.974842 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:06:58.977987 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.03s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.289
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.624
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.241
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.132
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.508
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.753
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.146
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.394
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.401
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.215
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.584
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.786
INFO:tensorflow:Finished evaluation at 2020-07-25-10:07:00
I0725 10:07:00.203981 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:07:00
INFO:tensorflow:Saving dict for global step 24034: DetectionBoxes_Precision/mAP = 0.28894165, DetectionBoxes_Precision/mAP (large) = 0.7527327, DetectionBoxes_Precision/mAP (medium) = 0.5080396, DetectionBoxes_Precision/mAP (small) = 0.13230048, DetectionBoxes_Precision/mAP@.50IOU = 0.62422234, DetectionBoxes_Precision/mAP@.75IOU = 0.24054705, DetectionBoxes_Recall/AR@1 = 0.14622754, DetectionBoxes_Recall/AR@10 = 0.3936527, DetectionBoxes_Recall/AR@100 = 0.4009581, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.5838196, DetectionBoxes_Recall/AR@100 (small) = 0.21488371, Loss/classification_loss = 3.035004, Loss/localization_loss = 0.71767503, Loss/regularization_loss = 0.51185375, Loss/total_loss = 4.2645326, global_step = 24034, learning_rate = 0.003, loss = 4.2645326
I0725 10:07:00.204360 139696487446400 estimator.py:2049] Saving dict for global step 24034: DetectionBoxes_Precision/mAP = 0.28894165, DetectionBoxes_Precision/mAP (large) = 0.7527327, DetectionBoxes_Precision/mAP (medium) = 0.5080396, DetectionBoxes_Precision/mAP (small) = 0.13230048, DetectionBoxes_Precision/mAP@.50IOU = 0.62422234, DetectionBoxes_Precision/mAP@.75IOU = 0.24054705, DetectionBoxes_Recall/AR@1 = 0.14622754, DetectionBoxes_Recall/AR@10 = 0.3936527, DetectionBoxes_Recall/AR@100 = 0.4009581, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.5838196, DetectionBoxes_Recall/AR@100 (small) = 0.21488371, Loss/classification_loss = 3.035004, Loss/localization_loss = 0.71767503, Loss/regularization_loss = 0.51185375, Loss/total_loss = 4.2645326, global_step = 24034, learning_rate = 0.003, loss = 4.2645326
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 24034: training/model.ckpt-24034
I0725 10:07:00.208703 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 24034: training/model.ckpt-24034
INFO:tensorflow:global_step/sec: 1.38702
I0725 10:07:34.387205 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38702
INFO:tensorflow:loss = 2.5857482, step = 24100 (72.098 sec)
I0725 10:07:34.388638 139696487446400 basic_session_run_hooks.py:260] loss = 2.5857482, step = 24100 (72.098 sec)
INFO:tensorflow:global_step/sec: 1.97563
I0725 10:08:25.003907 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97563
INFO:tensorflow:loss = 3.2571359, step = 24200 (50.617 sec)
I0725 10:08:25.005260 139696487446400 basic_session_run_hooks.py:260] loss = 3.2571359, step = 24200 (50.617 sec)
INFO:tensorflow:global_step/sec: 1.96378
I0725 10:09:15.926233 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96378
INFO:tensorflow:loss = 2.8795605, step = 24300 (50.922 sec)
I0725 10:09:15.927514 139696487446400 basic_session_run_hooks.py:260] loss = 2.8795605, step = 24300 (50.922 sec)
INFO:tensorflow:global_step/sec: 1.97539
I0725 10:10:06.549310 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97539
INFO:tensorflow:loss = 3.7724185, step = 24400 (50.623 sec)
I0725 10:10:06.550754 139696487446400 basic_session_run_hooks.py:260] loss = 3.7724185, step = 24400 (50.623 sec)
INFO:tensorflow:global_step/sec: 1.96717
I0725 10:10:57.383861 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96717
INFO:tensorflow:loss = 2.2212195, step = 24500 (50.834 sec)
I0725 10:10:57.385203 139696487446400 basic_session_run_hooks.py:260] loss = 2.2212195, step = 24500 (50.834 sec)
INFO:tensorflow:global_step/sec: 1.97517
I0725 10:11:48.012324 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97517
INFO:tensorflow:loss = 2.613522, step = 24600 (50.629 sec)
I0725 10:11:48.013750 139696487446400 basic_session_run_hooks.py:260] loss = 2.613522, step = 24600 (50.629 sec)
INFO:tensorflow:global_step/sec: 1.97328
I0725 10:12:38.689406 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97328
INFO:tensorflow:loss = 2.6096334, step = 24700 (50.677 sec)
I0725 10:12:38.690539 139696487446400 basic_session_run_hooks.py:260] loss = 2.6096334, step = 24700 (50.677 sec)
INFO:tensorflow:global_step/sec: 1.98901
I0725 10:13:28.965839 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98901
INFO:tensorflow:loss = 3.009624, step = 24800 (50.277 sec)
I0725 10:13:28.967215 139696487446400 basic_session_run_hooks.py:260] loss = 3.009624, step = 24800 (50.277 sec)
INFO:tensorflow:global_step/sec: 1.99559
I0725 10:14:19.076317 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99559
INFO:tensorflow:loss = 3.4342518, step = 24900 (50.110 sec)
I0725 10:14:19.077504 139696487446400 basic_session_run_hooks.py:260] loss = 3.4342518, step = 24900 (50.110 sec)
INFO:tensorflow:global_step/sec: 1.96889
I0725 10:15:09.866285 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96889
INFO:tensorflow:loss = 1.820425, step = 25000 (50.790 sec)
I0725 10:15:09.867777 139696487446400 basic_session_run_hooks.py:260] loss = 1.820425, step = 25000 (50.790 sec)
INFO:tensorflow:global_step/sec: 1.98044
I0725 10:16:00.360236 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98044
INFO:tensorflow:loss = 2.7319078, step = 25100 (50.494 sec)
I0725 10:16:00.361486 139696487446400 basic_session_run_hooks.py:260] loss = 2.7319078, step = 25100 (50.494 sec)
INFO:tensorflow:Saving checkpoints for 25178 into training/model.ckpt.
I0725 10:16:39.019692 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 25178 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3f92588>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:16:40.689428 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3f92588>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3fe48c8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:16:40.916440 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3fe48c8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:16:41.571288 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.367429 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.409989 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.451381 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.506761 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.551962 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:16:44.593089 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:16:46.906347 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:16:46Z
I0725 10:16:46.927957 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:16:46Z
INFO:tensorflow:Graph was finalized.
I0725 10:16:47.490218 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:16:47.491099: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:16:47.491757: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:16:47.491896: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:16:47.491966: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:16:47.492031: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:16:47.492089: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:16:47.492150: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:16:47.492211: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:16:47.492273: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:16:47.492431: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:16:47.493139: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:16:47.493673: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:16:47.493834: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:16:47.493864: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:16:47.493882: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:16:47.494055: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:16:47.494682: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:16:47.495319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-25178
I0725 10:16:47.496688 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-25178
INFO:tensorflow:Running local_init_op.
I0725 10:16:48.504909 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:16:48.653872 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:17:00.128292 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:17:00.129999 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:17:00.133334 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.04s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.316
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.667
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.243
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.162
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.481
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.724
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.143
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.394
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.402
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.231
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.569
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.779
INFO:tensorflow:Finished evaluation at 2020-07-25-10:17:01
I0725 10:17:01.399295 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:17:01
INFO:tensorflow:Saving dict for global step 25178: DetectionBoxes_Precision/mAP = 0.31594276, DetectionBoxes_Precision/mAP (large) = 0.72416425, DetectionBoxes_Precision/mAP (medium) = 0.48140162, DetectionBoxes_Precision/mAP (small) = 0.16179234, DetectionBoxes_Precision/mAP@.50IOU = 0.6665528, DetectionBoxes_Precision/mAP@.75IOU = 0.24313544, DetectionBoxes_Recall/AR@1 = 0.14299402, DetectionBoxes_Recall/AR@10 = 0.3942515, DetectionBoxes_Recall/AR@100 = 0.4021557, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.56870025, DetectionBoxes_Recall/AR@100 (small) = 0.23093024, Loss/classification_loss = 3.0963538, Loss/localization_loss = 0.7035841, Loss/regularization_loss = 0.49949884, Loss/total_loss = 4.299438, global_step = 25178, learning_rate = 0.003, loss = 4.299438
I0725 10:17:01.399595 139696487446400 estimator.py:2049] Saving dict for global step 25178: DetectionBoxes_Precision/mAP = 0.31594276, DetectionBoxes_Precision/mAP (large) = 0.72416425, DetectionBoxes_Precision/mAP (medium) = 0.48140162, DetectionBoxes_Precision/mAP (small) = 0.16179234, DetectionBoxes_Precision/mAP@.50IOU = 0.6665528, DetectionBoxes_Precision/mAP@.75IOU = 0.24313544, DetectionBoxes_Recall/AR@1 = 0.14299402, DetectionBoxes_Recall/AR@10 = 0.3942515, DetectionBoxes_Recall/AR@100 = 0.4021557, DetectionBoxes_Recall/AR@100 (large) = 0.77931035, DetectionBoxes_Recall/AR@100 (medium) = 0.56870025, DetectionBoxes_Recall/AR@100 (small) = 0.23093024, Loss/classification_loss = 3.0963538, Loss/localization_loss = 0.7035841, Loss/regularization_loss = 0.49949884, Loss/total_loss = 4.299438, global_step = 25178, learning_rate = 0.003, loss = 4.299438
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 25178: training/model.ckpt-25178
I0725 10:17:01.404074 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 25178: training/model.ckpt-25178
INFO:tensorflow:global_step/sec: 1.37446
I0725 10:17:13.116256 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.37446
INFO:tensorflow:loss = 2.8760867, step = 25200 (72.756 sec)
I0725 10:17:13.117562 139696487446400 basic_session_run_hooks.py:260] loss = 2.8760867, step = 25200 (72.756 sec)
INFO:tensorflow:global_step/sec: 1.9766
I0725 10:18:03.708097 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9766
INFO:tensorflow:loss = 2.7775187, step = 25300 (50.592 sec)
I0725 10:18:03.709160 139696487446400 basic_session_run_hooks.py:260] loss = 2.7775187, step = 25300 (50.592 sec)
INFO:tensorflow:global_step/sec: 1.96877
I0725 10:18:54.501190 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96877
INFO:tensorflow:loss = 1.9329884, step = 25400 (50.793 sec)
I0725 10:18:54.502177 139696487446400 basic_session_run_hooks.py:260] loss = 1.9329884, step = 25400 (50.793 sec)
INFO:tensorflow:global_step/sec: 1.95871
I0725 10:19:45.555234 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95871
INFO:tensorflow:loss = 2.115796, step = 25500 (51.055 sec)
I0725 10:19:45.556878 139696487446400 basic_session_run_hooks.py:260] loss = 2.115796, step = 25500 (51.055 sec)
INFO:tensorflow:global_step/sec: 1.95778
I0725 10:20:36.633522 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95778
INFO:tensorflow:loss = 2.8859403, step = 25600 (51.078 sec)
I0725 10:20:36.634590 139696487446400 basic_session_run_hooks.py:260] loss = 2.8859403, step = 25600 (51.078 sec)
INFO:tensorflow:global_step/sec: 1.95507
I0725 10:21:27.782763 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95507
INFO:tensorflow:loss = 2.288236, step = 25700 (51.149 sec)
I0725 10:21:27.783884 139696487446400 basic_session_run_hooks.py:260] loss = 2.288236, step = 25700 (51.149 sec)
INFO:tensorflow:global_step/sec: 1.95653
I0725 10:22:18.893731 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95653
INFO:tensorflow:loss = 2.8990145, step = 25800 (51.111 sec)
I0725 10:22:18.895082 139696487446400 basic_session_run_hooks.py:260] loss = 2.8990145, step = 25800 (51.111 sec)
INFO:tensorflow:global_step/sec: 1.94805
I0725 10:23:10.227043 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94805
INFO:tensorflow:loss = 2.710638, step = 25900 (51.333 sec)
I0725 10:23:10.228200 139696487446400 basic_session_run_hooks.py:260] loss = 2.710638, step = 25900 (51.333 sec)
INFO:tensorflow:global_step/sec: 1.94228
I0725 10:24:01.713017 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94228
INFO:tensorflow:loss = 3.3342645, step = 26000 (51.486 sec)
I0725 10:24:01.714377 139696487446400 basic_session_run_hooks.py:260] loss = 3.3342645, step = 26000 (51.486 sec)
INFO:tensorflow:global_step/sec: 1.9318
I0725 10:24:53.478222 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9318
INFO:tensorflow:loss = 2.2366214, step = 26100 (51.765 sec)
I0725 10:24:53.479692 139696487446400 basic_session_run_hooks.py:260] loss = 2.2366214, step = 26100 (51.765 sec)
INFO:tensorflow:global_step/sec: 1.98108
I0725 10:25:43.955756 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98108
INFO:tensorflow:loss = 2.1571248, step = 26200 (50.478 sec)
I0725 10:25:43.957192 139696487446400 basic_session_run_hooks.py:260] loss = 2.1571248, step = 26200 (50.478 sec)
INFO:tensorflow:global_step/sec: 1.97017
I0725 10:26:34.712688 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97017
INFO:tensorflow:loss = 3.5183747, step = 26300 (50.757 sec)
I0725 10:26:34.713798 139696487446400 basic_session_run_hooks.py:260] loss = 3.5183747, step = 26300 (50.757 sec)
INFO:tensorflow:Saving checkpoints for 26310 into training/model.ckpt.
I0725 10:26:39.172720 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 26310 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0ccf734518>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:26:40.864630 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0ccf734518>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd18597b8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:26:41.093595 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd18597b8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:26:41.755075 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.511709 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.553007 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.593489 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.636086 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.676571 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:26:44.716763 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:26:46.567328 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:26:46Z
I0725 10:26:46.588218 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:26:46Z
INFO:tensorflow:Graph was finalized.
I0725 10:26:47.171022 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:26:47.171897: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:26:47.172512: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:26:47.172663: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:26:47.172728: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:26:47.172786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:26:47.172868: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:26:47.172924: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:26:47.172973: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:26:47.173023: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:26:47.173173: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:26:47.173830: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:26:47.174369: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:26:47.174431: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:26:47.174458: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:26:47.174475: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:26:47.174642: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:26:47.175321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:26:47.175938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-26310
I0725 10:26:47.177310 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-26310
INFO:tensorflow:Running local_init_op.
I0725 10:26:48.136833 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:26:48.263721 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:26:59.568486 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:26:59.570015 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:26:59.573597 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.09s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.298
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.634
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.232
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.463
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.718
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.140
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.371
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.379
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.213
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.539
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.772
INFO:tensorflow:Finished evaluation at 2020-07-25-10:27:00
I0725 10:27:00.862051 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:27:00
INFO:tensorflow:Saving dict for global step 26310: DetectionBoxes_Precision/mAP = 0.2982758, DetectionBoxes_Precision/mAP (large) = 0.71824884, DetectionBoxes_Precision/mAP (medium) = 0.46324274, DetectionBoxes_Precision/mAP (small) = 0.14603212, DetectionBoxes_Precision/mAP@.50IOU = 0.63425356, DetectionBoxes_Precision/mAP@.75IOU = 0.23227891, DetectionBoxes_Recall/AR@1 = 0.13976048, DetectionBoxes_Recall/AR@10 = 0.37113774, DetectionBoxes_Recall/AR@100 = 0.3794012, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.53899205, DetectionBoxes_Recall/AR@100 (small) = 0.21325581, Loss/classification_loss = 3.268728, Loss/localization_loss = 0.7810642, Loss/regularization_loss = 0.48796904, Loss/total_loss = 4.5377626, global_step = 26310, learning_rate = 0.003, loss = 4.5377626
I0725 10:27:00.862410 139696487446400 estimator.py:2049] Saving dict for global step 26310: DetectionBoxes_Precision/mAP = 0.2982758, DetectionBoxes_Precision/mAP (large) = 0.71824884, DetectionBoxes_Precision/mAP (medium) = 0.46324274, DetectionBoxes_Precision/mAP (small) = 0.14603212, DetectionBoxes_Precision/mAP@.50IOU = 0.63425356, DetectionBoxes_Precision/mAP@.75IOU = 0.23227891, DetectionBoxes_Recall/AR@1 = 0.13976048, DetectionBoxes_Recall/AR@10 = 0.37113774, DetectionBoxes_Recall/AR@100 = 0.3794012, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.53899205, DetectionBoxes_Recall/AR@100 (small) = 0.21325581, Loss/classification_loss = 3.268728, Loss/localization_loss = 0.7810642, Loss/regularization_loss = 0.48796904, Loss/total_loss = 4.5377626, global_step = 26310, learning_rate = 0.003, loss = 4.5377626
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 26310: training/model.ckpt-26310
I0725 10:27:00.866631 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 26310: training/model.ckpt-26310
INFO:tensorflow:global_step/sec: 1.38615
I0725 10:27:46.854882 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38615
INFO:tensorflow:loss = 2.7858968, step = 26400 (72.142 sec)
I0725 10:27:46.856221 139696487446400 basic_session_run_hooks.py:260] loss = 2.7858968, step = 26400 (72.142 sec)
INFO:tensorflow:global_step/sec: 1.97624
I0725 10:28:37.456257 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97624
INFO:tensorflow:loss = 2.1934075, step = 26500 (50.601 sec)
I0725 10:28:37.457556 139696487446400 basic_session_run_hooks.py:260] loss = 2.1934075, step = 26500 (50.601 sec)
INFO:tensorflow:global_step/sec: 1.9706
I0725 10:29:28.202191 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9706
INFO:tensorflow:loss = 2.7976747, step = 26600 (50.746 sec)
I0725 10:29:28.203417 139696487446400 basic_session_run_hooks.py:260] loss = 2.7976747, step = 26600 (50.746 sec)
INFO:tensorflow:global_step/sec: 1.93962
I0725 10:30:19.758699 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93962
INFO:tensorflow:loss = 2.2823544, step = 26700 (51.557 sec)
I0725 10:30:19.760224 139696487446400 basic_session_run_hooks.py:260] loss = 2.2823544, step = 26700 (51.557 sec)
INFO:tensorflow:global_step/sec: 1.93973
I0725 10:31:11.312295 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93973
INFO:tensorflow:loss = 2.3028696, step = 26800 (51.553 sec)
I0725 10:31:11.313641 139696487446400 basic_session_run_hooks.py:260] loss = 2.3028696, step = 26800 (51.553 sec)
INFO:tensorflow:global_step/sec: 1.9542
I0725 10:32:02.484179 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9542
INFO:tensorflow:loss = 2.6225977, step = 26900 (51.172 sec)
I0725 10:32:02.485398 139696487446400 basic_session_run_hooks.py:260] loss = 2.6225977, step = 26900 (51.172 sec)
INFO:tensorflow:global_step/sec: 1.93987
I0725 10:32:54.034142 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93987
INFO:tensorflow:loss = 3.231905, step = 27000 (51.550 sec)
I0725 10:32:54.035423 139696487446400 basic_session_run_hooks.py:260] loss = 3.231905, step = 27000 (51.550 sec)
INFO:tensorflow:global_step/sec: 1.96152
I0725 10:33:45.015027 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96152
INFO:tensorflow:loss = 2.8039804, step = 27100 (50.981 sec)
I0725 10:33:45.016166 139696487446400 basic_session_run_hooks.py:260] loss = 2.8039804, step = 27100 (50.981 sec)
INFO:tensorflow:global_step/sec: 1.95746
I0725 10:34:36.101773 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95746
INFO:tensorflow:loss = 2.198093, step = 27200 (51.087 sec)
I0725 10:34:36.103220 139696487446400 basic_session_run_hooks.py:260] loss = 2.198093, step = 27200 (51.087 sec)
INFO:tensorflow:global_step/sec: 1.95866
I0725 10:35:27.157150 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95866
INFO:tensorflow:loss = 2.2041187, step = 27300 (51.056 sec)
I0725 10:35:27.159432 139696487446400 basic_session_run_hooks.py:260] loss = 2.2041187, step = 27300 (51.056 sec)
INFO:tensorflow:global_step/sec: 1.94507
I0725 10:36:18.569176 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94507
INFO:tensorflow:loss = 1.8847809, step = 27400 (51.411 sec)
I0725 10:36:18.570526 139696487446400 basic_session_run_hooks.py:260] loss = 1.8847809, step = 27400 (51.411 sec)
INFO:tensorflow:Saving checkpoints for 27442 into training/model.ckpt.
I0725 10:36:39.636487 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 27442 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d201de4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:36:41.319304 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d201de4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d201edb70> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:36:41.540427 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d201edb70> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:36:42.184015 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.360580 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.400950 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.439949 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.480630 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.520107 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:36:45.560756 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:36:47.456226 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:36:47Z
I0725 10:36:47.479064 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:36:47Z
INFO:tensorflow:Graph was finalized.
I0725 10:36:48.042074 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:36:48.042975: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:36:48.043695: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:36:48.043866: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:36:48.043937: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:36:48.043989: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:36:48.044041: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:36:48.044090: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:36:48.044134: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:36:48.044179: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:36:48.044336: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:36:48.045083: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:36:48.045627: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:36:48.045773: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:36:48.045803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:36:48.045842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:36:48.046018: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:36:48.046650: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:36:48.047217: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-27442
I0725 10:36:48.048454 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-27442
INFO:tensorflow:Running local_init_op.
I0725 10:36:49.029207 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:36:49.172498 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:37:00.633748 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:37:00.637517 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:37:00.640544 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.09s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.327
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.667
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.268
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.159
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.522
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.739
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.151
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.408
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.414
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.228
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.599
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.769
INFO:tensorflow:Finished evaluation at 2020-07-25-10:37:01
I0725 10:37:01.944288 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:37:01
INFO:tensorflow:Saving dict for global step 27442: DetectionBoxes_Precision/mAP = 0.3266757, DetectionBoxes_Precision/mAP (large) = 0.7386457, DetectionBoxes_Precision/mAP (medium) = 0.5221805, DetectionBoxes_Precision/mAP (small) = 0.15912226, DetectionBoxes_Precision/mAP@.50IOU = 0.6669897, DetectionBoxes_Precision/mAP@.75IOU = 0.2677565, DetectionBoxes_Recall/AR@1 = 0.15077844, DetectionBoxes_Recall/AR@10 = 0.40790418, DetectionBoxes_Recall/AR@100 = 0.4142515, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.598939, DetectionBoxes_Recall/AR@100 (small) = 0.2283721, Loss/classification_loss = 2.945757, Loss/localization_loss = 0.6881443, Loss/regularization_loss = 0.47702995, Loss/total_loss = 4.1109333, global_step = 27442, learning_rate = 0.003, loss = 4.1109333
I0725 10:37:01.944616 139696487446400 estimator.py:2049] Saving dict for global step 27442: DetectionBoxes_Precision/mAP = 0.3266757, DetectionBoxes_Precision/mAP (large) = 0.7386457, DetectionBoxes_Precision/mAP (medium) = 0.5221805, DetectionBoxes_Precision/mAP (small) = 0.15912226, DetectionBoxes_Precision/mAP@.50IOU = 0.6669897, DetectionBoxes_Precision/mAP@.75IOU = 0.2677565, DetectionBoxes_Recall/AR@1 = 0.15077844, DetectionBoxes_Recall/AR@10 = 0.40790418, DetectionBoxes_Recall/AR@100 = 0.4142515, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.598939, DetectionBoxes_Recall/AR@100 (small) = 0.2283721, Loss/classification_loss = 2.945757, Loss/localization_loss = 0.6881443, Loss/regularization_loss = 0.47702995, Loss/total_loss = 4.1109333, global_step = 27442, learning_rate = 0.003, loss = 4.1109333
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 27442: training/model.ckpt-27442
I0725 10:37:01.948770 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 27442: training/model.ckpt-27442
INFO:tensorflow:global_step/sec: 1.3495
I0725 10:37:32.670443 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3495
INFO:tensorflow:loss = 2.1852555, step = 27500 (74.101 sec)
I0725 10:37:32.671890 139696487446400 basic_session_run_hooks.py:260] loss = 2.1852555, step = 27500 (74.101 sec)
INFO:tensorflow:global_step/sec: 1.9205
I0725 10:38:24.740110 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9205
INFO:tensorflow:loss = 1.9489255, step = 27600 (52.069 sec)
I0725 10:38:24.741278 139696487446400 basic_session_run_hooks.py:260] loss = 1.9489255, step = 27600 (52.069 sec)
INFO:tensorflow:global_step/sec: 1.9392
I0725 10:39:16.307862 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9392
INFO:tensorflow:loss = 2.5377584, step = 27700 (51.568 sec)
I0725 10:39:16.309246 139696487446400 basic_session_run_hooks.py:260] loss = 2.5377584, step = 27700 (51.568 sec)
INFO:tensorflow:global_step/sec: 1.94712
I0725 10:40:07.665747 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94712
INFO:tensorflow:loss = 2.9463694, step = 27800 (51.358 sec)
I0725 10:40:07.667131 139696487446400 basic_session_run_hooks.py:260] loss = 2.9463694, step = 27800 (51.358 sec)
INFO:tensorflow:global_step/sec: 1.96982
I0725 10:40:58.431703 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96982
INFO:tensorflow:loss = 2.8994188, step = 27900 (50.766 sec)
I0725 10:40:58.433279 139696487446400 basic_session_run_hooks.py:260] loss = 2.8994188, step = 27900 (50.766 sec)
INFO:tensorflow:global_step/sec: 1.9687
I0725 10:41:49.226706 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9687
INFO:tensorflow:loss = 2.6755264, step = 28000 (50.795 sec)
I0725 10:41:49.227970 139696487446400 basic_session_run_hooks.py:260] loss = 2.6755264, step = 28000 (50.795 sec)
INFO:tensorflow:global_step/sec: 1.98166
I0725 10:42:39.689438 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98166
INFO:tensorflow:loss = 2.2737405, step = 28100 (50.464 sec)
I0725 10:42:39.692189 139696487446400 basic_session_run_hooks.py:260] loss = 2.2737405, step = 28100 (50.464 sec)
INFO:tensorflow:global_step/sec: 1.95916
I0725 10:43:30.731724 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95916
INFO:tensorflow:loss = 2.4570932, step = 28200 (51.041 sec)
I0725 10:43:30.733189 139696487446400 basic_session_run_hooks.py:260] loss = 2.4570932, step = 28200 (51.041 sec)
INFO:tensorflow:global_step/sec: 1.96419
I0725 10:44:21.643383 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96419
INFO:tensorflow:loss = 2.6860614, step = 28300 (50.912 sec)
I0725 10:44:21.644720 139696487446400 basic_session_run_hooks.py:260] loss = 2.6860614, step = 28300 (50.912 sec)
INFO:tensorflow:global_step/sec: 1.9454
I0725 10:45:13.046604 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9454
INFO:tensorflow:loss = 2.2217245, step = 28400 (51.403 sec)
I0725 10:45:13.047965 139696487446400 basic_session_run_hooks.py:260] loss = 2.2217245, step = 28400 (51.403 sec)
INFO:tensorflow:global_step/sec: 1.93363
I0725 10:46:04.762830 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93363
INFO:tensorflow:loss = 2.4118674, step = 28500 (51.716 sec)
I0725 10:46:04.763985 139696487446400 basic_session_run_hooks.py:260] loss = 2.4118674, step = 28500 (51.716 sec)
INFO:tensorflow:Saving checkpoints for 28569 into training/model.ckpt.
I0725 10:46:39.661252 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 28569 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd35414a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:46:41.465423 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd35414a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd35201e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:46:41.699989 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd35201e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:46:42.388118 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.280405 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.321900 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.363107 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.404421 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.444753 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:46:45.486013 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:46:47.793416 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:46:47Z
I0725 10:46:47.815642 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:46:47Z
INFO:tensorflow:Graph was finalized.
I0725 10:46:48.408163 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:46:48.409069: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:46:48.409681: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:46:48.409878: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:46:48.409939: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:46:48.410008: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:46:48.410060: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:46:48.410109: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:46:48.410164: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:46:48.410219: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:46:48.410370: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:46:48.411041: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:46:48.411593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:46:48.411657: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:46:48.411687: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:46:48.411704: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:46:48.411918: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:46:48.412590: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:46:48.413167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-28569
I0725 10:46:48.414528 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-28569
INFO:tensorflow:Running local_init_op.
I0725 10:46:49.507214 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:46:49.663049 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:47:01.625791 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:47:01.627305 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:47:01.630584 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.14s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.319
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.656
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.161
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.500
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.713
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.142
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.397
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.406
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.232
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.576
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.772
INFO:tensorflow:Finished evaluation at 2020-07-25-10:47:02
I0725 10:47:02.987752 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:47:02
INFO:tensorflow:Saving dict for global step 28569: DetectionBoxes_Precision/mAP = 0.31878793, DetectionBoxes_Precision/mAP (large) = 0.71318024, DetectionBoxes_Precision/mAP (medium) = 0.5001964, DetectionBoxes_Precision/mAP (small) = 0.16097216, DetectionBoxes_Precision/mAP@.50IOU = 0.65568143, DetectionBoxes_Precision/mAP@.75IOU = 0.25204825, DetectionBoxes_Recall/AR@1 = 0.14239521, DetectionBoxes_Recall/AR@10 = 0.3966467, DetectionBoxes_Recall/AR@100 = 0.405509, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.57586205, DetectionBoxes_Recall/AR@100 (small) = 0.23162791, Loss/classification_loss = 3.0911386, Loss/localization_loss = 0.6927459, Loss/regularization_loss = 0.46704313, Loss/total_loss = 4.2509284, global_step = 28569, learning_rate = 0.003, loss = 4.2509284
I0725 10:47:02.988248 139696487446400 estimator.py:2049] Saving dict for global step 28569: DetectionBoxes_Precision/mAP = 0.31878793, DetectionBoxes_Precision/mAP (large) = 0.71318024, DetectionBoxes_Precision/mAP (medium) = 0.5001964, DetectionBoxes_Precision/mAP (small) = 0.16097216, DetectionBoxes_Precision/mAP@.50IOU = 0.65568143, DetectionBoxes_Precision/mAP@.75IOU = 0.25204825, DetectionBoxes_Recall/AR@1 = 0.14239521, DetectionBoxes_Recall/AR@10 = 0.3966467, DetectionBoxes_Recall/AR@100 = 0.405509, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.57586205, DetectionBoxes_Recall/AR@100 (small) = 0.23162791, Loss/classification_loss = 3.0911386, Loss/localization_loss = 0.6927459, Loss/regularization_loss = 0.46704313, Loss/total_loss = 4.2509284, global_step = 28569, learning_rate = 0.003, loss = 4.2509284
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 28569: training/model.ckpt-28569
I0725 10:47:02.993260 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 28569: training/model.ckpt-28569
INFO:tensorflow:global_step/sec: 1.33947
I0725 10:47:19.418929 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.33947
INFO:tensorflow:loss = 3.3078861, step = 28600 (74.656 sec)
I0725 10:47:19.420104 139696487446400 basic_session_run_hooks.py:260] loss = 3.3078861, step = 28600 (74.656 sec)
INFO:tensorflow:global_step/sec: 1.96022
I0725 10:48:10.433708 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96022
INFO:tensorflow:loss = 2.169634, step = 28700 (51.015 sec)
I0725 10:48:10.435425 139696487446400 basic_session_run_hooks.py:260] loss = 2.169634, step = 28700 (51.015 sec)
INFO:tensorflow:global_step/sec: 1.959
I0725 10:49:01.480372 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.959
INFO:tensorflow:loss = 2.6935763, step = 28800 (51.047 sec)
I0725 10:49:01.482538 139696487446400 basic_session_run_hooks.py:260] loss = 2.6935763, step = 28800 (51.047 sec)
INFO:tensorflow:global_step/sec: 1.93158
I0725 10:49:53.251331 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93158
INFO:tensorflow:loss = 3.1038446, step = 28900 (51.770 sec)
I0725 10:49:53.252679 139696487446400 basic_session_run_hooks.py:260] loss = 3.1038446, step = 28900 (51.770 sec)
INFO:tensorflow:global_step/sec: 1.93269
I0725 10:50:44.992529 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93269
INFO:tensorflow:loss = 2.542596, step = 29000 (51.741 sec)
I0725 10:50:44.993806 139696487446400 basic_session_run_hooks.py:260] loss = 2.542596, step = 29000 (51.741 sec)
INFO:tensorflow:global_step/sec: 1.9538
I0725 10:51:36.174954 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9538
INFO:tensorflow:loss = 3.5874789, step = 29100 (51.182 sec)
I0725 10:51:36.176053 139696487446400 basic_session_run_hooks.py:260] loss = 3.5874789, step = 29100 (51.182 sec)
INFO:tensorflow:global_step/sec: 1.92977
I0725 10:52:27.994504 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.92977
INFO:tensorflow:loss = 3.5896497, step = 29200 (51.820 sec)
I0725 10:52:27.995972 139696487446400 basic_session_run_hooks.py:260] loss = 3.5896497, step = 29200 (51.820 sec)
INFO:tensorflow:global_step/sec: 1.94872
I0725 10:53:19.310182 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94872
INFO:tensorflow:loss = 2.6662924, step = 29300 (51.316 sec)
I0725 10:53:19.311484 139696487446400 basic_session_run_hooks.py:260] loss = 2.6662924, step = 29300 (51.316 sec)
INFO:tensorflow:global_step/sec: 1.93873
I0725 10:54:10.890328 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93873
INFO:tensorflow:loss = 2.613587, step = 29400 (51.581 sec)
I0725 10:54:10.892099 139696487446400 basic_session_run_hooks.py:260] loss = 2.613587, step = 29400 (51.581 sec)
INFO:tensorflow:global_step/sec: 1.93872
I0725 10:55:02.470679 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93872
INFO:tensorflow:loss = 2.3510783, step = 29500 (51.580 sec)
I0725 10:55:02.472224 139696487446400 basic_session_run_hooks.py:260] loss = 2.3510783, step = 29500 (51.580 sec)
INFO:tensorflow:global_step/sec: 1.96761
I0725 10:55:53.293873 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96761
INFO:tensorflow:loss = 2.153019, step = 29600 (50.823 sec)
I0725 10:55:53.295209 139696487446400 basic_session_run_hooks.py:260] loss = 2.153019, step = 29600 (50.823 sec)
INFO:tensorflow:Saving checkpoints for 29692 into training/model.ckpt.
I0725 10:56:39.699549 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 29692 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d201ab4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 10:56:41.511492 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d201ab4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2cd46a8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 10:56:41.757297 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2cd46a8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 10:56:42.466363 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.323521 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.398671 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.444758 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.489781 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.531869 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 10:56:45.573439 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 10:56:47.568526 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T10:56:47Z
I0725 10:56:47.591747 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T10:56:47Z
INFO:tensorflow:Graph was finalized.
I0725 10:56:48.187329 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 10:56:48.188266: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:56:48.189010: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 10:56:48.189116: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 10:56:48.189186: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 10:56:48.189245: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 10:56:48.189299: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 10:56:48.189350: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 10:56:48.189409: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 10:56:48.189460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 10:56:48.189617: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:56:48.190507: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:56:48.191119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 10:56:48.191279: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 10:56:48.191313: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 10:56:48.191331: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 10:56:48.191519: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:56:48.192220: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 10:56:48.192795: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-29692
I0725 10:56:48.194272 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-29692
INFO:tensorflow:Running local_init_op.
I0725 10:56:49.230150 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 10:56:49.374791 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 10:57:01.166503 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 10:57:01.168874 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 10:57:01.170707 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.16s).
Accumulating evaluation results...
DONE (t=0.12s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.323
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.651
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.260
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.161
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.488
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.713
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.390
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.396
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.218
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.569
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.772
INFO:tensorflow:Finished evaluation at 2020-07-25-10:57:02
I0725 10:57:02.560027 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-10:57:02
INFO:tensorflow:Saving dict for global step 29692: DetectionBoxes_Precision/mAP = 0.3232151, DetectionBoxes_Precision/mAP (large) = 0.7125984, DetectionBoxes_Precision/mAP (medium) = 0.4878529, DetectionBoxes_Precision/mAP (small) = 0.1606633, DetectionBoxes_Precision/mAP@.50IOU = 0.6508281, DetectionBoxes_Precision/mAP@.75IOU = 0.26007387, DetectionBoxes_Recall/AR@1 = 0.14742514, DetectionBoxes_Recall/AR@10 = 0.39041916, DetectionBoxes_Recall/AR@100 = 0.39556885, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.5692308, DetectionBoxes_Recall/AR@100 (small) = 0.21790698, Loss/classification_loss = 3.282504, Loss/localization_loss = 0.7227743, Loss/regularization_loss = 0.4576494, Loss/total_loss = 4.46293, global_step = 29692, learning_rate = 0.003, loss = 4.46293
I0725 10:57:02.560522 139696487446400 estimator.py:2049] Saving dict for global step 29692: DetectionBoxes_Precision/mAP = 0.3232151, DetectionBoxes_Precision/mAP (large) = 0.7125984, DetectionBoxes_Precision/mAP (medium) = 0.4878529, DetectionBoxes_Precision/mAP (small) = 0.1606633, DetectionBoxes_Precision/mAP@.50IOU = 0.6508281, DetectionBoxes_Precision/mAP@.75IOU = 0.26007387, DetectionBoxes_Recall/AR@1 = 0.14742514, DetectionBoxes_Recall/AR@10 = 0.39041916, DetectionBoxes_Recall/AR@100 = 0.39556885, DetectionBoxes_Recall/AR@100 (large) = 0.7724138, DetectionBoxes_Recall/AR@100 (medium) = 0.5692308, DetectionBoxes_Recall/AR@100 (small) = 0.21790698, Loss/classification_loss = 3.282504, Loss/localization_loss = 0.7227743, Loss/regularization_loss = 0.4576494, Loss/total_loss = 4.46293, global_step = 29692, learning_rate = 0.003, loss = 4.46293
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 29692: training/model.ckpt-29692
I0725 10:57:02.566585 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 29692: training/model.ckpt-29692
INFO:tensorflow:global_step/sec: 1.3537
I0725 10:57:07.165297 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3537
INFO:tensorflow:loss = 2.533237, step = 29700 (73.871 sec)
I0725 10:57:07.166655 139696487446400 basic_session_run_hooks.py:260] loss = 2.533237, step = 29700 (73.871 sec)
INFO:tensorflow:global_step/sec: 1.96774
I0725 10:57:57.985114 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96774
INFO:tensorflow:loss = 2.5801888, step = 29800 (50.820 sec)
I0725 10:57:57.986433 139696487446400 basic_session_run_hooks.py:260] loss = 2.5801888, step = 29800 (50.820 sec)
INFO:tensorflow:global_step/sec: 1.93487
I0725 10:58:49.668246 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93487
INFO:tensorflow:loss = 2.0024738, step = 29900 (51.683 sec)
I0725 10:58:49.669759 139696487446400 basic_session_run_hooks.py:260] loss = 2.0024738, step = 29900 (51.683 sec)
INFO:tensorflow:global_step/sec: 1.93507
I0725 10:59:41.345844 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93507
INFO:tensorflow:loss = 2.24511, step = 30000 (51.677 sec)
I0725 10:59:41.347108 139696487446400 basic_session_run_hooks.py:260] loss = 2.24511, step = 30000 (51.677 sec)
INFO:tensorflow:global_step/sec: 1.94142
I0725 11:00:32.854465 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94142
INFO:tensorflow:loss = 3.6884012, step = 30100 (51.509 sec)
I0725 11:00:32.856119 139696487446400 basic_session_run_hooks.py:260] loss = 3.6884012, step = 30100 (51.509 sec)
INFO:tensorflow:global_step/sec: 1.94441
I0725 11:01:24.284047 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94441
INFO:tensorflow:loss = 2.182117, step = 30200 (51.429 sec)
I0725 11:01:24.285388 139696487446400 basic_session_run_hooks.py:260] loss = 2.182117, step = 30200 (51.429 sec)
INFO:tensorflow:global_step/sec: 1.96648
I0725 11:02:15.136220 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96648
INFO:tensorflow:loss = 2.0008595, step = 30300 (50.852 sec)
I0725 11:02:15.137468 139696487446400 basic_session_run_hooks.py:260] loss = 2.0008595, step = 30300 (50.852 sec)
INFO:tensorflow:global_step/sec: 1.96645
I0725 11:03:05.989396 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96645
INFO:tensorflow:loss = 2.3767447, step = 30400 (50.854 sec)
I0725 11:03:05.990988 139696487446400 basic_session_run_hooks.py:260] loss = 2.3767447, step = 30400 (50.854 sec)
INFO:tensorflow:global_step/sec: 1.97293
I0725 11:03:56.675475 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97293
INFO:tensorflow:loss = 2.486345, step = 30500 (50.686 sec)
I0725 11:03:56.676761 139696487446400 basic_session_run_hooks.py:260] loss = 2.486345, step = 30500 (50.686 sec)
INFO:tensorflow:global_step/sec: 1.9441
I0725 11:04:48.113154 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9441
INFO:tensorflow:loss = 2.0013964, step = 30600 (51.438 sec)
I0725 11:04:48.114409 139696487446400 basic_session_run_hooks.py:260] loss = 2.0013964, step = 30600 (51.438 sec)
INFO:tensorflow:global_step/sec: 1.95867
I0725 11:05:39.168084 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95867
INFO:tensorflow:loss = 2.935659, step = 30700 (51.055 sec)
I0725 11:05:39.169443 139696487446400 basic_session_run_hooks.py:260] loss = 2.935659, step = 30700 (51.055 sec)
INFO:tensorflow:global_step/sec: 1.95657
I0725 11:06:30.278059 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95657
INFO:tensorflow:loss = 3.4364047, step = 30800 (51.110 sec)
I0725 11:06:30.279422 139696487446400 basic_session_run_hooks.py:260] loss = 3.4364047, step = 30800 (51.110 sec)
INFO:tensorflow:Saving checkpoints for 30820 into training/model.ckpt.
I0725 11:06:40.052013 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 30820 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd388e0f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:06:41.804510 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd388e0f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3949c80> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:06:42.041712 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3949c80> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:06:42.738453 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.039641 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.084111 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.126806 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.170143 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.213986 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:06:46.266117 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:06:48.233678 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:06:48Z
I0725 11:06:48.255323 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:06:48Z
INFO:tensorflow:Graph was finalized.
I0725 11:06:48.838145 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:06:48.839050: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:06:48.839709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:06:48.839897: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:06:48.839958: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:06:48.840020: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:06:48.840073: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:06:48.840123: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:06:48.840172: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:06:48.840240: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:06:48.840409: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:06:48.841126: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:06:48.841681: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:06:48.841745: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:06:48.841772: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:06:48.841805: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:06:48.842002: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:06:48.842655: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:06:48.843281: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-30820
I0725 11:06:48.844747 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-30820
INFO:tensorflow:Running local_init_op.
I0725 11:06:49.901979 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:06:50.052390 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:07:01.938770 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:07:01.940561 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:07:01.942654 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.18s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.329
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.671
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.275
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.164
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.500
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.760
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.149
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.396
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.404
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.222
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.582
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-11:07:03
I0725 11:07:03.318547 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:07:03
INFO:tensorflow:Saving dict for global step 30820: DetectionBoxes_Precision/mAP = 0.32898775, DetectionBoxes_Precision/mAP (large) = 0.76033443, DetectionBoxes_Precision/mAP (medium) = 0.49957713, DetectionBoxes_Precision/mAP (small) = 0.1639024, DetectionBoxes_Precision/mAP@.50IOU = 0.67133886, DetectionBoxes_Precision/mAP@.75IOU = 0.27525648, DetectionBoxes_Recall/AR@1 = 0.1491018, DetectionBoxes_Recall/AR@10 = 0.39592814, DetectionBoxes_Recall/AR@100 = 0.40407187, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5822281, DetectionBoxes_Recall/AR@100 (small) = 0.2216279, Loss/classification_loss = 3.026333, Loss/localization_loss = 0.7129799, Loss/regularization_loss = 0.44865158, Loss/total_loss = 4.1879625, global_step = 30820, learning_rate = 0.003, loss = 4.1879625
I0725 11:07:03.318928 139696487446400 estimator.py:2049] Saving dict for global step 30820: DetectionBoxes_Precision/mAP = 0.32898775, DetectionBoxes_Precision/mAP (large) = 0.76033443, DetectionBoxes_Precision/mAP (medium) = 0.49957713, DetectionBoxes_Precision/mAP (small) = 0.1639024, DetectionBoxes_Precision/mAP@.50IOU = 0.67133886, DetectionBoxes_Precision/mAP@.75IOU = 0.27525648, DetectionBoxes_Recall/AR@1 = 0.1491018, DetectionBoxes_Recall/AR@10 = 0.39592814, DetectionBoxes_Recall/AR@100 = 0.40407187, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5822281, DetectionBoxes_Recall/AR@100 (small) = 0.2216279, Loss/classification_loss = 3.026333, Loss/localization_loss = 0.7129799, Loss/regularization_loss = 0.44865158, Loss/total_loss = 4.1879625, global_step = 30820, learning_rate = 0.003, loss = 4.1879625
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 30820: training/model.ckpt-30820
I0725 11:07:03.323448 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 30820: training/model.ckpt-30820
INFO:tensorflow:global_step/sec: 1.33928
I0725 11:07:44.945064 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.33928
INFO:tensorflow:loss = 2.7329583, step = 30900 (74.668 sec)
I0725 11:07:44.947220 139696487446400 basic_session_run_hooks.py:260] loss = 2.7329583, step = 30900 (74.668 sec)
INFO:tensorflow:global_step/sec: 1.93832
I0725 11:08:36.535992 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93832
INFO:tensorflow:loss = 2.8443217, step = 31000 (51.590 sec)
I0725 11:08:36.537277 139696487446400 basic_session_run_hooks.py:260] loss = 2.8443217, step = 31000 (51.590 sec)
INFO:tensorflow:global_step/sec: 1.95793
I0725 11:09:27.610283 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95793
INFO:tensorflow:loss = 2.0903819, step = 31100 (51.074 sec)
I0725 11:09:27.611554 139696487446400 basic_session_run_hooks.py:260] loss = 2.0903819, step = 31100 (51.074 sec)
INFO:tensorflow:global_step/sec: 1.94314
I0725 11:10:19.073437 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94314
INFO:tensorflow:loss = 2.5693836, step = 31200 (51.463 sec)
I0725 11:10:19.075038 139696487446400 basic_session_run_hooks.py:260] loss = 2.5693836, step = 31200 (51.463 sec)
INFO:tensorflow:global_step/sec: 1.93299
I0725 11:11:10.806745 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93299
INFO:tensorflow:loss = 2.294083, step = 31300 (51.733 sec)
I0725 11:11:10.808206 139696487446400 basic_session_run_hooks.py:260] loss = 2.294083, step = 31300 (51.733 sec)
INFO:tensorflow:global_step/sec: 1.94933
I0725 11:12:02.106317 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94933
INFO:tensorflow:loss = 1.9851092, step = 31400 (51.299 sec)
I0725 11:12:02.107688 139696487446400 basic_session_run_hooks.py:260] loss = 1.9851092, step = 31400 (51.299 sec)
INFO:tensorflow:global_step/sec: 1.97816
I0725 11:12:52.658465 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97816
INFO:tensorflow:loss = 2.722851, step = 31500 (50.552 sec)
I0725 11:12:52.660008 139696487446400 basic_session_run_hooks.py:260] loss = 2.722851, step = 31500 (50.552 sec)
INFO:tensorflow:global_step/sec: 1.98674
I0725 11:13:42.992085 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98674
INFO:tensorflow:loss = 2.841922, step = 31600 (50.333 sec)
I0725 11:13:42.993210 139696487446400 basic_session_run_hooks.py:260] loss = 2.841922, step = 31600 (50.333 sec)
INFO:tensorflow:global_step/sec: 1.98499
I0725 11:14:33.370239 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98499
INFO:tensorflow:loss = 1.7312354, step = 31700 (50.378 sec)
I0725 11:14:33.371458 139696487446400 basic_session_run_hooks.py:260] loss = 1.7312354, step = 31700 (50.378 sec)
INFO:tensorflow:global_step/sec: 1.9721
I0725 11:15:24.077687 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9721
INFO:tensorflow:loss = 2.2741964, step = 31800 (50.708 sec)
I0725 11:15:24.079239 139696487446400 basic_session_run_hooks.py:260] loss = 2.2741964, step = 31800 (50.708 sec)
INFO:tensorflow:global_step/sec: 1.95617
I0725 11:16:15.198079 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95617
INFO:tensorflow:loss = 2.851786, step = 31900 (51.120 sec)
I0725 11:16:15.199365 139696487446400 basic_session_run_hooks.py:260] loss = 2.851786, step = 31900 (51.120 sec)
INFO:tensorflow:Saving checkpoints for 31950 into training/model.ckpt.
I0725 11:16:40.342681 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 31950 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd39009b0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:16:42.140260 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd39009b0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2c671e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:16:42.373179 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd2c671e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:16:43.078099 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:45.978355 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:46.020923 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:46.062259 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:46.113155 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:46.153542 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:16:46.201058 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:16:48.131728 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:16:48Z
I0725 11:16:48.153860 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:16:48Z
INFO:tensorflow:Graph was finalized.
I0725 11:16:49.077770 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:16:49.078646: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:16:49.079411: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:16:49.079518: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:16:49.079580: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:16:49.079646: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:16:49.079710: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:16:49.079802: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:16:49.079903: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:16:49.079956: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:16:49.080113: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:16:49.080892: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:16:49.081462: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:16:49.081630: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:16:49.081660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:16:49.081676: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:16:49.081907: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:16:49.082632: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:16:49.083288: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-31950
I0725 11:16:49.084727 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-31950
INFO:tensorflow:Running local_init_op.
I0725 11:16:50.129547 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:16:50.288835 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:17:02.202568 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:17:02.204406 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:17:02.208317 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.18s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.296
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.645
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.224
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.137
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.498
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.750
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.143
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.387
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.394
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.222
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.559
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-11:17:03
I0725 11:17:03.584749 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:17:03
INFO:tensorflow:Saving dict for global step 31950: DetectionBoxes_Precision/mAP = 0.29568225, DetectionBoxes_Precision/mAP (large) = 0.75011295, DetectionBoxes_Precision/mAP (medium) = 0.49826315, DetectionBoxes_Precision/mAP (small) = 0.13663702, DetectionBoxes_Precision/mAP@.50IOU = 0.64543945, DetectionBoxes_Precision/mAP@.75IOU = 0.22375776, DetectionBoxes_Recall/AR@1 = 0.14287426, DetectionBoxes_Recall/AR@10 = 0.38730538, DetectionBoxes_Recall/AR@100 = 0.39389223, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5591512, DetectionBoxes_Recall/AR@100 (small) = 0.22209302, Loss/classification_loss = 3.1049337, Loss/localization_loss = 0.70896477, Loss/regularization_loss = 0.44044426, Loss/total_loss = 4.254342, global_step = 31950, learning_rate = 0.003, loss = 4.254342
I0725 11:17:03.585117 139696487446400 estimator.py:2049] Saving dict for global step 31950: DetectionBoxes_Precision/mAP = 0.29568225, DetectionBoxes_Precision/mAP (large) = 0.75011295, DetectionBoxes_Precision/mAP (medium) = 0.49826315, DetectionBoxes_Precision/mAP (small) = 0.13663702, DetectionBoxes_Precision/mAP@.50IOU = 0.64543945, DetectionBoxes_Precision/mAP@.75IOU = 0.22375776, DetectionBoxes_Recall/AR@1 = 0.14287426, DetectionBoxes_Recall/AR@10 = 0.38730538, DetectionBoxes_Recall/AR@100 = 0.39389223, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5591512, DetectionBoxes_Recall/AR@100 (small) = 0.22209302, Loss/classification_loss = 3.1049337, Loss/localization_loss = 0.70896477, Loss/regularization_loss = 0.44044426, Loss/total_loss = 4.254342, global_step = 31950, learning_rate = 0.003, loss = 4.254342
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 31950: training/model.ckpt-31950
I0725 11:17:03.589712 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 31950: training/model.ckpt-31950
INFO:tensorflow:global_step/sec: 1.33811
I0725 11:17:29.930539 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.33811
INFO:tensorflow:loss = 3.292786, step = 32000 (74.733 sec)
I0725 11:17:29.931910 139696487446400 basic_session_run_hooks.py:260] loss = 3.292786, step = 32000 (74.733 sec)
INFO:tensorflow:global_step/sec: 1.95158
I0725 11:18:21.170982 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95158
INFO:tensorflow:loss = 2.7412405, step = 32100 (51.240 sec)
I0725 11:18:21.172213 139696487446400 basic_session_run_hooks.py:260] loss = 2.7412405, step = 32100 (51.240 sec)
INFO:tensorflow:global_step/sec: 1.94761
I0725 11:19:12.516001 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94761
INFO:tensorflow:loss = 2.472633, step = 32200 (51.345 sec)
I0725 11:19:12.517455 139696487446400 basic_session_run_hooks.py:260] loss = 2.472633, step = 32200 (51.345 sec)
INFO:tensorflow:global_step/sec: 1.94247
I0725 11:20:03.996786 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94247
INFO:tensorflow:loss = 2.289388, step = 32300 (51.481 sec)
I0725 11:20:03.998382 139696487446400 basic_session_run_hooks.py:260] loss = 2.289388, step = 32300 (51.481 sec)
INFO:tensorflow:global_step/sec: 1.96761
I0725 11:20:54.819880 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96761
INFO:tensorflow:loss = 2.2118256, step = 32400 (50.823 sec)
I0725 11:20:54.821278 139696487446400 basic_session_run_hooks.py:260] loss = 2.2118256, step = 32400 (50.823 sec)
INFO:tensorflow:global_step/sec: 1.96259
I0725 11:21:45.772924 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96259
INFO:tensorflow:loss = 2.265201, step = 32500 (50.953 sec)
I0725 11:21:45.774126 139696487446400 basic_session_run_hooks.py:260] loss = 2.265201, step = 32500 (50.953 sec)
INFO:tensorflow:global_step/sec: 1.95045
I0725 11:22:37.043046 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95045
INFO:tensorflow:loss = 3.4180784, step = 32600 (51.270 sec)
I0725 11:22:37.044324 139696487446400 basic_session_run_hooks.py:260] loss = 3.4180784, step = 32600 (51.270 sec)
INFO:tensorflow:global_step/sec: 1.99199
I0725 11:23:27.244062 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99199
INFO:tensorflow:loss = 2.6236997, step = 32700 (50.201 sec)
I0725 11:23:27.245775 139696487446400 basic_session_run_hooks.py:260] loss = 2.6236997, step = 32700 (50.201 sec)
INFO:tensorflow:global_step/sec: 1.97143
I0725 11:24:17.968529 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97143
INFO:tensorflow:loss = 1.9476695, step = 32800 (50.724 sec)
I0725 11:24:17.969937 139696487446400 basic_session_run_hooks.py:260] loss = 1.9476695, step = 32800 (50.724 sec)
INFO:tensorflow:global_step/sec: 1.96261
I0725 11:25:08.921079 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96261
INFO:tensorflow:loss = 2.9799259, step = 32900 (50.953 sec)
I0725 11:25:08.922524 139696487446400 basic_session_run_hooks.py:260] loss = 2.9799259, step = 32900 (50.953 sec)
INFO:tensorflow:global_step/sec: 1.97765
I0725 11:25:59.486183 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97765
INFO:tensorflow:loss = 2.261258, step = 33000 (50.565 sec)
I0725 11:25:59.487784 139696487446400 basic_session_run_hooks.py:260] loss = 2.261258, step = 33000 (50.565 sec)
INFO:tensorflow:Saving checkpoints for 33082 into training/model.ckpt.
I0725 11:26:40.759257 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 33082 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d21c32908>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:26:42.508004 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d21c32908>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3129268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:26:42.743191 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3129268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:26:43.440777 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.376082 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.418130 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.459694 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.500966 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.542322 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:26:46.604612 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:26:48.542791 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:26:48Z
I0725 11:26:48.566480 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:26:48Z
INFO:tensorflow:Graph was finalized.
I0725 11:26:49.145187 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:26:49.146027: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:26:49.146653: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:26:49.146802: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:26:49.146884: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:26:49.146946: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:26:49.147025: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:26:49.147109: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:26:49.147161: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:26:49.147214: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:26:49.147379: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:26:49.148132: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:26:49.148729: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:26:49.148793: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:26:49.148837: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:26:49.148864: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:26:49.149036: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:26:49.149751: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:26:49.150328: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-33082
I0725 11:26:49.151579 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-33082
INFO:tensorflow:Running local_init_op.
I0725 11:26:50.172223 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:26:50.322239 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:27:02.209002 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:27:02.210390 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:27:02.212422 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.17s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.318
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.648
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.286
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.520
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.724
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.149
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.400
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.407
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.221
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.589
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.769
INFO:tensorflow:Finished evaluation at 2020-07-25-11:27:03
I0725 11:27:03.589967 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:27:03
INFO:tensorflow:Saving dict for global step 33082: DetectionBoxes_Precision/mAP = 0.31767395, DetectionBoxes_Precision/mAP (large) = 0.7244844, DetectionBoxes_Precision/mAP (medium) = 0.5199828, DetectionBoxes_Precision/mAP (small) = 0.14598568, DetectionBoxes_Precision/mAP@.50IOU = 0.6481741, DetectionBoxes_Precision/mAP@.75IOU = 0.28565663, DetectionBoxes_Recall/AR@1 = 0.1491018, DetectionBoxes_Recall/AR@10 = 0.3996407, DetectionBoxes_Recall/AR@100 = 0.40658683, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5893899, DetectionBoxes_Recall/AR@100 (small) = 0.22093023, Loss/classification_loss = 3.0775652, Loss/localization_loss = 0.6771405, Loss/regularization_loss = 0.43269122, Loss/total_loss = 4.187396, global_step = 33082, learning_rate = 0.003, loss = 4.187396
I0725 11:27:03.590292 139696487446400 estimator.py:2049] Saving dict for global step 33082: DetectionBoxes_Precision/mAP = 0.31767395, DetectionBoxes_Precision/mAP (large) = 0.7244844, DetectionBoxes_Precision/mAP (medium) = 0.5199828, DetectionBoxes_Precision/mAP (small) = 0.14598568, DetectionBoxes_Precision/mAP@.50IOU = 0.6481741, DetectionBoxes_Precision/mAP@.75IOU = 0.28565663, DetectionBoxes_Recall/AR@1 = 0.1491018, DetectionBoxes_Recall/AR@10 = 0.3996407, DetectionBoxes_Recall/AR@100 = 0.40658683, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5893899, DetectionBoxes_Recall/AR@100 (small) = 0.22093023, Loss/classification_loss = 3.0775652, Loss/localization_loss = 0.6771405, Loss/regularization_loss = 0.43269122, Loss/total_loss = 4.187396, global_step = 33082, learning_rate = 0.003, loss = 4.187396
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 33082: training/model.ckpt-33082
I0725 11:27:03.594774 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 33082: training/model.ckpt-33082
INFO:tensorflow:global_step/sec: 1.35454
I0725 11:27:13.312165 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.35454
INFO:tensorflow:loss = 2.7430098, step = 33100 (73.826 sec)
I0725 11:27:13.313384 139696487446400 basic_session_run_hooks.py:260] loss = 2.7430098, step = 33100 (73.826 sec)
INFO:tensorflow:global_step/sec: 1.95929
I0725 11:28:04.351162 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95929
INFO:tensorflow:loss = 2.3591018, step = 33200 (51.039 sec)
I0725 11:28:04.352779 139696487446400 basic_session_run_hooks.py:260] loss = 2.3591018, step = 33200 (51.039 sec)
INFO:tensorflow:global_step/sec: 1.9797
I0725 11:28:54.863736 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9797
INFO:tensorflow:loss = 2.1829283, step = 33300 (50.512 sec)
I0725 11:28:54.865204 139696487446400 basic_session_run_hooks.py:260] loss = 2.1829283, step = 33300 (50.512 sec)
INFO:tensorflow:global_step/sec: 1.97094
I0725 11:29:45.600912 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97094
INFO:tensorflow:loss = 2.2176101, step = 33400 (50.737 sec)
I0725 11:29:45.602508 139696487446400 basic_session_run_hooks.py:260] loss = 2.2176101, step = 33400 (50.737 sec)
INFO:tensorflow:global_step/sec: 1.9719
I0725 11:30:36.313398 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9719
INFO:tensorflow:loss = 2.1653314, step = 33500 (50.712 sec)
I0725 11:30:36.314674 139696487446400 basic_session_run_hooks.py:260] loss = 2.1653314, step = 33500 (50.712 sec)
INFO:tensorflow:global_step/sec: 1.96684
I0725 11:31:27.156477 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96684
INFO:tensorflow:loss = 2.1414242, step = 33600 (50.843 sec)
I0725 11:31:27.157806 139696487446400 basic_session_run_hooks.py:260] loss = 2.1414242, step = 33600 (50.843 sec)
INFO:tensorflow:global_step/sec: 1.98494
I0725 11:32:17.535853 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98494
INFO:tensorflow:loss = 2.0746472, step = 33700 (50.380 sec)
I0725 11:32:17.537329 139696487446400 basic_session_run_hooks.py:260] loss = 2.0746472, step = 33700 (50.380 sec)
INFO:tensorflow:global_step/sec: 1.96207
I0725 11:33:08.502545 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96207
INFO:tensorflow:loss = 1.5995331, step = 33800 (50.967 sec)
I0725 11:33:08.504132 139696487446400 basic_session_run_hooks.py:260] loss = 1.5995331, step = 33800 (50.967 sec)
INFO:tensorflow:global_step/sec: 1.95857
I0725 11:33:59.560191 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95857
INFO:tensorflow:loss = 3.4177542, step = 33900 (51.057 sec)
I0725 11:33:59.561412 139696487446400 basic_session_run_hooks.py:260] loss = 3.4177542, step = 33900 (51.057 sec)
INFO:tensorflow:global_step/sec: 1.9821
I0725 11:34:50.011900 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9821
INFO:tensorflow:loss = 2.430905, step = 34000 (50.452 sec)
I0725 11:34:50.013258 139696487446400 basic_session_run_hooks.py:260] loss = 2.430905, step = 34000 (50.452 sec)
INFO:tensorflow:global_step/sec: 1.96916
I0725 11:35:40.794890 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96916
INFO:tensorflow:loss = 2.5093024, step = 34100 (50.783 sec)
I0725 11:35:40.796634 139696487446400 basic_session_run_hooks.py:260] loss = 2.5093024, step = 34100 (50.783 sec)
INFO:tensorflow:global_step/sec: 1.97837
I0725 11:36:31.341588 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97837
INFO:tensorflow:loss = 2.5414317, step = 34200 (50.546 sec)
I0725 11:36:31.343084 139696487446400 basic_session_run_hooks.py:260] loss = 2.5414317, step = 34200 (50.546 sec)
INFO:tensorflow:Saving checkpoints for 34220 into training/model.ckpt.
I0725 11:36:40.986522 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 34220 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3cbb4e0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:36:42.697758 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3cbb4e0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd404de18> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:36:42.936539 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd404de18> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:36:43.623194 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:46.915095 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:46.966142 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:47.007838 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:47.050900 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:47.093661 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:36:47.135004 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:36:49.113395 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:36:49Z
I0725 11:36:49.135136 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:36:49Z
INFO:tensorflow:Graph was finalized.
I0725 11:36:49.716688 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:36:49.717570: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:36:49.718241: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:36:49.718352: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:36:49.718421: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:36:49.718496: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:36:49.718568: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:36:49.718639: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:36:49.718717: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:36:49.718797: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:36:49.719023: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:36:49.719875: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:36:49.720431: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:36:49.720590: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:36:49.720625: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:36:49.720643: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:36:49.720843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:36:49.721493: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:36:49.722075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-34220
I0725 11:36:49.723538 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-34220
INFO:tensorflow:Running local_init_op.
I0725 11:36:50.761255 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:36:50.928149 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:37:02.762591 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:37:02.764064 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:37:02.765902 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.14s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.332
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.665
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.288
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.153
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.511
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.761
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.399
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.403
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.220
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.582
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.800
INFO:tensorflow:Finished evaluation at 2020-07-25-11:37:04
I0725 11:37:04.124377 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:37:04
INFO:tensorflow:Saving dict for global step 34220: DetectionBoxes_Precision/mAP = 0.3319638, DetectionBoxes_Precision/mAP (large) = 0.76139456, DetectionBoxes_Precision/mAP (medium) = 0.51064, DetectionBoxes_Precision/mAP (small) = 0.15347196, DetectionBoxes_Precision/mAP@.50IOU = 0.6645931, DetectionBoxes_Precision/mAP@.75IOU = 0.28803948, DetectionBoxes_Recall/AR@1 = 0.14682634, DetectionBoxes_Recall/AR@10 = 0.39892215, DetectionBoxes_Recall/AR@100 = 0.40323353, DetectionBoxes_Recall/AR@100 (large) = 0.8, DetectionBoxes_Recall/AR@100 (medium) = 0.5822281, DetectionBoxes_Recall/AR@100 (small) = 0.21953489, Loss/classification_loss = 3.1168249, Loss/localization_loss = 0.7121631, Loss/regularization_loss = 0.42545262, Loss/total_loss = 4.25444, global_step = 34220, learning_rate = 0.003, loss = 4.25444
I0725 11:37:04.124714 139696487446400 estimator.py:2049] Saving dict for global step 34220: DetectionBoxes_Precision/mAP = 0.3319638, DetectionBoxes_Precision/mAP (large) = 0.76139456, DetectionBoxes_Precision/mAP (medium) = 0.51064, DetectionBoxes_Precision/mAP (small) = 0.15347196, DetectionBoxes_Precision/mAP@.50IOU = 0.6645931, DetectionBoxes_Precision/mAP@.75IOU = 0.28803948, DetectionBoxes_Recall/AR@1 = 0.14682634, DetectionBoxes_Recall/AR@10 = 0.39892215, DetectionBoxes_Recall/AR@100 = 0.40323353, DetectionBoxes_Recall/AR@100 (large) = 0.8, DetectionBoxes_Recall/AR@100 (medium) = 0.5822281, DetectionBoxes_Recall/AR@100 (small) = 0.21953489, Loss/classification_loss = 3.1168249, Loss/localization_loss = 0.7121631, Loss/regularization_loss = 0.42545262, Loss/total_loss = 4.25444, global_step = 34220, learning_rate = 0.003, loss = 4.25444
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 34220: training/model.ckpt-34220
I0725 11:37:04.128948 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 34220: training/model.ckpt-34220
INFO:tensorflow:global_step/sec: 1.35166
I0725 11:37:45.324850 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.35166
INFO:tensorflow:loss = 2.717837, step = 34300 (73.983 sec)
I0725 11:37:45.326132 139696487446400 basic_session_run_hooks.py:260] loss = 2.717837, step = 34300 (73.983 sec)
INFO:tensorflow:global_step/sec: 1.96549
I0725 11:38:36.202713 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96549
INFO:tensorflow:loss = 1.9034164, step = 34400 (50.878 sec)
I0725 11:38:36.203904 139696487446400 basic_session_run_hooks.py:260] loss = 1.9034164, step = 34400 (50.878 sec)
INFO:tensorflow:global_step/sec: 1.96515
I0725 11:39:27.089421 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96515
INFO:tensorflow:loss = 2.6950698, step = 34500 (50.887 sec)
I0725 11:39:27.090854 139696487446400 basic_session_run_hooks.py:260] loss = 2.6950698, step = 34500 (50.887 sec)
INFO:tensorflow:global_step/sec: 1.97178
I0725 11:40:17.804891 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97178
INFO:tensorflow:loss = 2.5468435, step = 34600 (50.715 sec)
I0725 11:40:17.806215 139696487446400 basic_session_run_hooks.py:260] loss = 2.5468435, step = 34600 (50.715 sec)
INFO:tensorflow:global_step/sec: 1.96519
I0725 11:41:08.690579 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96519
INFO:tensorflow:loss = 2.7421227, step = 34700 (50.886 sec)
I0725 11:41:08.692030 139696487446400 basic_session_run_hooks.py:260] loss = 2.7421227, step = 34700 (50.886 sec)
INFO:tensorflow:global_step/sec: 1.96508
I0725 11:41:59.578996 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96508
INFO:tensorflow:loss = 2.182494, step = 34800 (50.888 sec)
I0725 11:41:59.580460 139696487446400 basic_session_run_hooks.py:260] loss = 2.182494, step = 34800 (50.888 sec)
INFO:tensorflow:global_step/sec: 1.9666
I0725 11:42:50.428077 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9666
INFO:tensorflow:loss = 2.5503333, step = 34900 (50.849 sec)
I0725 11:42:50.429593 139696487446400 basic_session_run_hooks.py:260] loss = 2.5503333, step = 34900 (50.849 sec)
INFO:tensorflow:global_step/sec: 1.97106
I0725 11:43:41.162271 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97106
INFO:tensorflow:loss = 1.991935, step = 35000 (50.734 sec)
I0725 11:43:41.163613 139696487446400 basic_session_run_hooks.py:260] loss = 1.991935, step = 35000 (50.734 sec)
INFO:tensorflow:global_step/sec: 1.97427
I0725 11:44:31.813802 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97427
INFO:tensorflow:loss = 3.3788207, step = 35100 (50.652 sec)
I0725 11:44:31.815261 139696487446400 basic_session_run_hooks.py:260] loss = 3.3788207, step = 35100 (50.652 sec)
INFO:tensorflow:global_step/sec: 1.98081
I0725 11:45:22.298225 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98081
INFO:tensorflow:loss = 2.0509486, step = 35200 (50.484 sec)
I0725 11:45:22.299587 139696487446400 basic_session_run_hooks.py:260] loss = 2.0509486, step = 35200 (50.484 sec)
INFO:tensorflow:global_step/sec: 1.98502
I0725 11:46:12.675452 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98502
INFO:tensorflow:loss = 2.4593103, step = 35300 (50.377 sec)
I0725 11:46:12.676900 139696487446400 basic_session_run_hooks.py:260] loss = 2.4593103, step = 35300 (50.377 sec)
INFO:tensorflow:Saving checkpoints for 35357 into training/model.ckpt.
I0725 11:46:41.283050 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 35357 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd05b8780>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:46:43.062242 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd05b8780>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd29f31e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:46:43.289347 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd29f31e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:46:43.964933 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:46.913298 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:46.954218 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:46.996169 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:47.039103 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:47.080217 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:46:47.121219 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:46:49.066176 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:46:49Z
I0725 11:46:49.089379 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:46:49Z
INFO:tensorflow:Graph was finalized.
I0725 11:46:49.672870 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:46:49.673660: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:46:49.674359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:46:49.674513: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:46:49.674585: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:46:49.674645: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:46:49.674716: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:46:49.674786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:46:49.674857: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:46:49.674919: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:46:49.675081: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:46:49.675783: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:46:49.676377: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:46:49.676454: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:46:49.676481: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:46:49.676502: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:46:49.676679: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:46:49.677364: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:46:49.677995: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-35357
I0725 11:46:49.679347 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-35357
INFO:tensorflow:Running local_init_op.
I0725 11:46:50.712712 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:46:50.854137 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:47:02.750273 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:47:02.751621 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:47:02.755409 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.19s).
Accumulating evaluation results...
DONE (t=0.07s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.298
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.651
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.213
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.150
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.468
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.740
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.141
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.382
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.388
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.225
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.545
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.762
INFO:tensorflow:Finished evaluation at 2020-07-25-11:47:04
I0725 11:47:04.123392 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:47:04
INFO:tensorflow:Saving dict for global step 35357: DetectionBoxes_Precision/mAP = 0.29825485, DetectionBoxes_Precision/mAP (large) = 0.73992544, DetectionBoxes_Precision/mAP (medium) = 0.46750665, DetectionBoxes_Precision/mAP (small) = 0.15045322, DetectionBoxes_Precision/mAP@.50IOU = 0.65146327, DetectionBoxes_Precision/mAP@.75IOU = 0.21250801, DetectionBoxes_Recall/AR@1 = 0.1405988, DetectionBoxes_Recall/AR@10 = 0.38179642, DetectionBoxes_Recall/AR@100 = 0.38778442, DetectionBoxes_Recall/AR@100 (large) = 0.762069, DetectionBoxes_Recall/AR@100 (medium) = 0.54535806, DetectionBoxes_Recall/AR@100 (small) = 0.22465116, Loss/classification_loss = 3.133145, Loss/localization_loss = 0.74360776, Loss/regularization_loss = 0.41864994, Loss/total_loss = 4.2954025, global_step = 35357, learning_rate = 0.003, loss = 4.2954025
I0725 11:47:04.123726 139696487446400 estimator.py:2049] Saving dict for global step 35357: DetectionBoxes_Precision/mAP = 0.29825485, DetectionBoxes_Precision/mAP (large) = 0.73992544, DetectionBoxes_Precision/mAP (medium) = 0.46750665, DetectionBoxes_Precision/mAP (small) = 0.15045322, DetectionBoxes_Precision/mAP@.50IOU = 0.65146327, DetectionBoxes_Precision/mAP@.75IOU = 0.21250801, DetectionBoxes_Recall/AR@1 = 0.1405988, DetectionBoxes_Recall/AR@10 = 0.38179642, DetectionBoxes_Recall/AR@100 = 0.38778442, DetectionBoxes_Recall/AR@100 (large) = 0.762069, DetectionBoxes_Recall/AR@100 (medium) = 0.54535806, DetectionBoxes_Recall/AR@100 (small) = 0.22465116, Loss/classification_loss = 3.133145, Loss/localization_loss = 0.74360776, Loss/regularization_loss = 0.41864994, Loss/total_loss = 4.2954025, global_step = 35357, learning_rate = 0.003, loss = 4.2954025
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 35357: training/model.ckpt-35357
I0725 11:47:04.128222 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 35357: training/model.ckpt-35357
INFO:tensorflow:global_step/sec: 1.35421
I0725 11:47:26.518969 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.35421
INFO:tensorflow:loss = 1.7534087, step = 35400 (73.843 sec)
I0725 11:47:26.520221 139696487446400 basic_session_run_hooks.py:260] loss = 1.7534087, step = 35400 (73.843 sec)
INFO:tensorflow:global_step/sec: 1.95781
I0725 11:48:17.596534 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95781
INFO:tensorflow:loss = 2.133896, step = 35500 (51.077 sec)
I0725 11:48:17.597696 139696487446400 basic_session_run_hooks.py:260] loss = 2.133896, step = 35500 (51.077 sec)
INFO:tensorflow:global_step/sec: 1.96261
I0725 11:49:08.549075 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96261
INFO:tensorflow:loss = 3.1563528, step = 35600 (50.953 sec)
I0725 11:49:08.550357 139696487446400 basic_session_run_hooks.py:260] loss = 3.1563528, step = 35600 (50.953 sec)
INFO:tensorflow:global_step/sec: 1.97663
I0725 11:49:59.140151 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97663
INFO:tensorflow:loss = 2.2848873, step = 35700 (50.591 sec)
I0725 11:49:59.141458 139696487446400 basic_session_run_hooks.py:260] loss = 2.2848873, step = 35700 (50.591 sec)
INFO:tensorflow:global_step/sec: 1.9638
I0725 11:50:50.061861 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9638
INFO:tensorflow:loss = 2.53083, step = 35800 (50.922 sec)
I0725 11:50:50.063384 139696487446400 basic_session_run_hooks.py:260] loss = 2.53083, step = 35800 (50.922 sec)
INFO:tensorflow:global_step/sec: 1.97993
I0725 11:51:40.568713 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97993
INFO:tensorflow:loss = 2.4100745, step = 35900 (50.507 sec)
I0725 11:51:40.570095 139696487446400 basic_session_run_hooks.py:260] loss = 2.4100745, step = 35900 (50.507 sec)
INFO:tensorflow:global_step/sec: 1.976
I0725 11:52:31.176198 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.976
INFO:tensorflow:loss = 2.425504, step = 36000 (50.608 sec)
I0725 11:52:31.178070 139696487446400 basic_session_run_hooks.py:260] loss = 2.425504, step = 36000 (50.608 sec)
INFO:tensorflow:global_step/sec: 1.97641
I0725 11:53:21.772968 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97641
INFO:tensorflow:loss = 2.4107103, step = 36100 (50.596 sec)
I0725 11:53:21.774423 139696487446400 basic_session_run_hooks.py:260] loss = 2.4107103, step = 36100 (50.596 sec)
INFO:tensorflow:global_step/sec: 1.97301
I0725 11:54:12.456795 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97301
INFO:tensorflow:loss = 2.2902021, step = 36200 (50.684 sec)
I0725 11:54:12.458098 139696487446400 basic_session_run_hooks.py:260] loss = 2.2902021, step = 36200 (50.684 sec)
INFO:tensorflow:global_step/sec: 1.97251
I0725 11:55:03.153463 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97251
INFO:tensorflow:loss = 2.8422139, step = 36300 (50.697 sec)
I0725 11:55:03.154774 139696487446400 basic_session_run_hooks.py:260] loss = 2.8422139, step = 36300 (50.697 sec)
INFO:tensorflow:global_step/sec: 1.96558
I0725 11:55:54.029202 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96558
INFO:tensorflow:loss = 2.0830045, step = 36400 (50.876 sec)
I0725 11:55:54.030545 139696487446400 basic_session_run_hooks.py:260] loss = 2.0830045, step = 36400 (50.876 sec)
INFO:tensorflow:Saving checkpoints for 36494 into training/model.ckpt.
I0725 11:56:41.435734 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 36494 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d200eb470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 11:56:43.211220 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d200eb470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d20259d90> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 11:56:43.456518 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d20259d90> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 11:56:44.549001 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.528973 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.573060 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.615436 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.657944 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.700848 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 11:56:47.742802 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 11:56:49.759355 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T11:56:49Z
I0725 11:56:49.782331 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T11:56:49Z
INFO:tensorflow:Graph was finalized.
I0725 11:56:50.372173 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 11:56:50.373106: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:56:50.373981: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 11:56:50.374089: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 11:56:50.374156: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 11:56:50.374215: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 11:56:50.374270: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 11:56:50.374321: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 11:56:50.374371: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 11:56:50.374422: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 11:56:50.374575: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:56:50.375282: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:56:50.375859: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 11:56:50.375998: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 11:56:50.376029: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 11:56:50.376047: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 11:56:50.376226: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:56:50.376940: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 11:56:50.377555: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-36494
I0725 11:56:50.379261 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-36494
INFO:tensorflow:Running local_init_op.
I0725 11:56:51.421704 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 11:56:51.574592 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 11:57:03.545454 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 11:57:03.547120 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 11:57:03.550651 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.15s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.299
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.656
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.222
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.136
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.501
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.756
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.144
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.389
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.396
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.217
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.569
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-11:57:04
I0725 11:57:04.917051 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-11:57:04
INFO:tensorflow:Saving dict for global step 36494: DetectionBoxes_Precision/mAP = 0.2985214, DetectionBoxes_Precision/mAP (large) = 0.7555596, DetectionBoxes_Precision/mAP (medium) = 0.5010045, DetectionBoxes_Precision/mAP (small) = 0.13562879, DetectionBoxes_Precision/mAP@.50IOU = 0.65634745, DetectionBoxes_Precision/mAP@.75IOU = 0.2217889, DetectionBoxes_Recall/AR@1 = 0.14419162, DetectionBoxes_Recall/AR@10 = 0.38898203, DetectionBoxes_Recall/AR@100 = 0.39568862, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.56870025, DetectionBoxes_Recall/AR@100 (small) = 0.21720931, Loss/classification_loss = 3.1434925, Loss/localization_loss = 0.71430314, Loss/regularization_loss = 0.41229305, Loss/total_loss = 4.2700877, global_step = 36494, learning_rate = 0.003, loss = 4.2700877
I0725 11:57:04.917393 139696487446400 estimator.py:2049] Saving dict for global step 36494: DetectionBoxes_Precision/mAP = 0.2985214, DetectionBoxes_Precision/mAP (large) = 0.7555596, DetectionBoxes_Precision/mAP (medium) = 0.5010045, DetectionBoxes_Precision/mAP (small) = 0.13562879, DetectionBoxes_Precision/mAP@.50IOU = 0.65634745, DetectionBoxes_Precision/mAP@.75IOU = 0.2217889, DetectionBoxes_Recall/AR@1 = 0.14419162, DetectionBoxes_Recall/AR@10 = 0.38898203, DetectionBoxes_Recall/AR@100 = 0.39568862, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.56870025, DetectionBoxes_Recall/AR@100 (small) = 0.21720931, Loss/classification_loss = 3.1434925, Loss/localization_loss = 0.71430314, Loss/regularization_loss = 0.41229305, Loss/total_loss = 4.2700877, global_step = 36494, learning_rate = 0.003, loss = 4.2700877
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 36494: training/model.ckpt-36494
I0725 11:57:04.922273 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 36494: training/model.ckpt-36494
INFO:tensorflow:global_step/sec: 1.34178
I0725 11:57:08.557154 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.34178
INFO:tensorflow:loss = 2.7802432, step = 36500 (74.528 sec)
I0725 11:57:08.558307 139696487446400 basic_session_run_hooks.py:260] loss = 2.7802432, step = 36500 (74.528 sec)
INFO:tensorflow:global_step/sec: 1.97182
I0725 11:57:59.271952 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97182
INFO:tensorflow:loss = 2.9587023, step = 36600 (50.715 sec)
I0725 11:57:59.273309 139696487446400 basic_session_run_hooks.py:260] loss = 2.9587023, step = 36600 (50.715 sec)
INFO:tensorflow:global_step/sec: 1.97848
I0725 11:58:49.815718 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97848
INFO:tensorflow:loss = 2.3715081, step = 36700 (50.544 sec)
I0725 11:58:49.817048 139696487446400 basic_session_run_hooks.py:260] loss = 2.3715081, step = 36700 (50.544 sec)
INFO:tensorflow:global_step/sec: 1.96895
I0725 11:59:40.604285 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96895
INFO:tensorflow:loss = 2.198895, step = 36800 (50.789 sec)
I0725 11:59:40.605968 139696487446400 basic_session_run_hooks.py:260] loss = 2.198895, step = 36800 (50.789 sec)
INFO:tensorflow:global_step/sec: 1.96945
I0725 12:00:31.379977 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96945
INFO:tensorflow:loss = 2.4206464, step = 36900 (50.775 sec)
I0725 12:00:31.381251 139696487446400 basic_session_run_hooks.py:260] loss = 2.4206464, step = 36900 (50.775 sec)
INFO:tensorflow:global_step/sec: 1.97296
I0725 12:01:22.065110 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97296
INFO:tensorflow:loss = 1.9708828, step = 37000 (50.685 sec)
I0725 12:01:22.066424 139696487446400 basic_session_run_hooks.py:260] loss = 1.9708828, step = 37000 (50.685 sec)
INFO:tensorflow:global_step/sec: 1.98085
I0725 12:02:12.548396 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98085
INFO:tensorflow:loss = 1.9581649, step = 37100 (50.483 sec)
I0725 12:02:12.549918 139696487446400 basic_session_run_hooks.py:260] loss = 1.9581649, step = 37100 (50.483 sec)
INFO:tensorflow:global_step/sec: 1.97529
I0725 12:03:03.174038 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97529
INFO:tensorflow:loss = 2.773976, step = 37200 (50.625 sec)
I0725 12:03:03.175409 139696487446400 basic_session_run_hooks.py:260] loss = 2.773976, step = 37200 (50.625 sec)
INFO:tensorflow:global_step/sec: 1.98967
I0725 12:03:53.433431 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98967
INFO:tensorflow:loss = 2.754575, step = 37300 (50.259 sec)
I0725 12:03:53.434532 139696487446400 basic_session_run_hooks.py:260] loss = 2.754575, step = 37300 (50.259 sec)
INFO:tensorflow:global_step/sec: 1.97695
I0725 12:04:44.016273 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97695
INFO:tensorflow:loss = 1.9468337, step = 37400 (50.583 sec)
I0725 12:04:44.017903 139696487446400 basic_session_run_hooks.py:260] loss = 1.9468337, step = 37400 (50.583 sec)
INFO:tensorflow:global_step/sec: 1.9684
I0725 12:05:34.819008 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9684
INFO:tensorflow:loss = 1.8854564, step = 37500 (50.804 sec)
I0725 12:05:34.821987 139696487446400 basic_session_run_hooks.py:260] loss = 1.8854564, step = 37500 (50.804 sec)
INFO:tensorflow:global_step/sec: 1.96492
I0725 12:06:25.711480 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96492
INFO:tensorflow:loss = 2.1249173, step = 37600 (50.891 sec)
I0725 12:06:25.712642 139696487446400 basic_session_run_hooks.py:260] loss = 2.1249173, step = 37600 (50.891 sec)
INFO:tensorflow:Saving checkpoints for 37633 into training/model.ckpt.
I0725 12:06:41.743685 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 37633 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3618cf8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:06:43.509285 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3618cf8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd35f57b8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:06:43.744378 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd35f57b8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:06:44.423453 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.273412 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.314565 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.355176 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.395942 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.436870 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:06:47.477678 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:06:49.741077 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:06:49Z
I0725 12:06:49.763488 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:06:49Z
INFO:tensorflow:Graph was finalized.
I0725 12:06:50.342949 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:06:50.343802: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:06:50.344443: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:06:50.344591: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:06:50.344648: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:06:50.344704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:06:50.344832: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:06:50.344897: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:06:50.344954: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:06:50.345007: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:06:50.345165: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:06:50.345866: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:06:50.346404: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:06:50.346466: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:06:50.346493: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:06:50.346510: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:06:50.346685: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:06:50.347364: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:06:50.347941: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-37633
I0725 12:06:50.349312 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-37633
INFO:tensorflow:Running local_init_op.
I0725 12:06:51.434064 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:06:51.625245 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:07:03.572016 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:07:03.573794 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:07:03.577327 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.12s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.310
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.656
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.250
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.141
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.521
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.754
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.145
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.400
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.406
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.227
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.579
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.803
INFO:tensorflow:Finished evaluation at 2020-07-25-12:07:04
I0725 12:07:04.916622 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:07:04
INFO:tensorflow:Saving dict for global step 37633: DetectionBoxes_Precision/mAP = 0.30981687, DetectionBoxes_Precision/mAP (large) = 0.75370866, DetectionBoxes_Precision/mAP (medium) = 0.5205811, DetectionBoxes_Precision/mAP (small) = 0.14092954, DetectionBoxes_Precision/mAP@.50IOU = 0.6558729, DetectionBoxes_Precision/mAP@.75IOU = 0.2500115, DetectionBoxes_Recall/AR@1 = 0.14491019, DetectionBoxes_Recall/AR@10 = 0.39952096, DetectionBoxes_Recall/AR@100 = 0.40562874, DetectionBoxes_Recall/AR@100 (large) = 0.80344826, DetectionBoxes_Recall/AR@100 (medium) = 0.5785146, DetectionBoxes_Recall/AR@100 (small) = 0.22744186, Loss/classification_loss = 3.1635299, Loss/localization_loss = 0.6917673, Loss/regularization_loss = 0.4061789, Loss/total_loss = 4.261475, global_step = 37633, learning_rate = 0.003, loss = 4.261475
I0725 12:07:04.917003 139696487446400 estimator.py:2049] Saving dict for global step 37633: DetectionBoxes_Precision/mAP = 0.30981687, DetectionBoxes_Precision/mAP (large) = 0.75370866, DetectionBoxes_Precision/mAP (medium) = 0.5205811, DetectionBoxes_Precision/mAP (small) = 0.14092954, DetectionBoxes_Precision/mAP@.50IOU = 0.6558729, DetectionBoxes_Precision/mAP@.75IOU = 0.2500115, DetectionBoxes_Recall/AR@1 = 0.14491019, DetectionBoxes_Recall/AR@10 = 0.39952096, DetectionBoxes_Recall/AR@100 = 0.40562874, DetectionBoxes_Recall/AR@100 (large) = 0.80344826, DetectionBoxes_Recall/AR@100 (medium) = 0.5785146, DetectionBoxes_Recall/AR@100 (small) = 0.22744186, Loss/classification_loss = 3.1635299, Loss/localization_loss = 0.6917673, Loss/regularization_loss = 0.4061789, Loss/total_loss = 4.261475, global_step = 37633, learning_rate = 0.003, loss = 4.261475
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 37633: training/model.ckpt-37633
I0725 12:07:04.921272 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 37633: training/model.ckpt-37633
INFO:tensorflow:global_step/sec: 1.34692
I0725 12:07:39.955100 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.34692
INFO:tensorflow:loss = 1.9151987, step = 37700 (74.244 sec)
I0725 12:07:39.956383 139696487446400 basic_session_run_hooks.py:260] loss = 1.9151987, step = 37700 (74.244 sec)
INFO:tensorflow:global_step/sec: 1.99436
I0725 12:08:30.096499 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99436
INFO:tensorflow:loss = 1.981877, step = 37800 (50.141 sec)
I0725 12:08:30.097887 139696487446400 basic_session_run_hooks.py:260] loss = 1.981877, step = 37800 (50.141 sec)
INFO:tensorflow:global_step/sec: 1.98476
I0725 12:09:20.480454 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98476
INFO:tensorflow:loss = 2.2405832, step = 37900 (50.384 sec)
I0725 12:09:20.482008 139696487446400 basic_session_run_hooks.py:260] loss = 2.2405832, step = 37900 (50.384 sec)
INFO:tensorflow:global_step/sec: 1.96031
I0725 12:10:11.492841 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96031
INFO:tensorflow:loss = 2.018386, step = 38000 (51.012 sec)
I0725 12:10:11.494411 139696487446400 basic_session_run_hooks.py:260] loss = 2.018386, step = 38000 (51.012 sec)
INFO:tensorflow:global_step/sec: 1.99604
I0725 12:11:01.592098 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99604
INFO:tensorflow:loss = 2.046921, step = 38100 (50.099 sec)
I0725 12:11:01.593402 139696487446400 basic_session_run_hooks.py:260] loss = 2.046921, step = 38100 (50.099 sec)
INFO:tensorflow:global_step/sec: 1.96614
I0725 12:11:52.453022 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96614
INFO:tensorflow:loss = 2.118364, step = 38200 (50.861 sec)
I0725 12:11:52.454330 139696487446400 basic_session_run_hooks.py:260] loss = 2.118364, step = 38200 (50.861 sec)
INFO:tensorflow:global_step/sec: 1.98511
I0725 12:12:42.828108 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98511
INFO:tensorflow:loss = 2.5762606, step = 38300 (50.375 sec)
I0725 12:12:42.829448 139696487446400 basic_session_run_hooks.py:260] loss = 2.5762606, step = 38300 (50.375 sec)
INFO:tensorflow:global_step/sec: 1.97453
I0725 12:13:33.473097 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97453
INFO:tensorflow:loss = 2.407595, step = 38400 (50.645 sec)
I0725 12:13:33.474433 139696487446400 basic_session_run_hooks.py:260] loss = 2.407595, step = 38400 (50.645 sec)
INFO:tensorflow:global_step/sec: 1.97619
I0725 12:14:24.075390 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97619
INFO:tensorflow:loss = 2.1937444, step = 38500 (50.602 sec)
I0725 12:14:24.076701 139696487446400 basic_session_run_hooks.py:260] loss = 2.1937444, step = 38500 (50.602 sec)
INFO:tensorflow:global_step/sec: 1.983
I0725 12:15:14.504179 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.983
INFO:tensorflow:loss = 2.9223313, step = 38600 (50.429 sec)
I0725 12:15:14.505840 139696487446400 basic_session_run_hooks.py:260] loss = 2.9223313, step = 38600 (50.429 sec)
INFO:tensorflow:global_step/sec: 1.9639
I0725 12:16:05.423156 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9639
INFO:tensorflow:loss = 1.7179375, step = 38700 (50.918 sec)
I0725 12:16:05.424272 139696487446400 basic_session_run_hooks.py:260] loss = 1.7179375, step = 38700 (50.918 sec)
INFO:tensorflow:Saving checkpoints for 38773 into training/model.ckpt.
I0725 12:16:42.083365 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 38773 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd574b6d8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:16:43.918212 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd574b6d8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd58181e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:16:44.157565 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd58181e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:16:44.838436 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.672492 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.715181 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.758253 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.809450 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.852697 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:16:47.894546 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:16:49.863779 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:16:49Z
I0725 12:16:49.885323 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:16:49Z
INFO:tensorflow:Graph was finalized.
I0725 12:16:50.466626 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:16:50.467544: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:16:50.468201: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:16:50.468313: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:16:50.468380: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:16:50.468444: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:16:50.468499: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:16:50.468558: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:16:50.468621: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:16:50.468680: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:16:50.468860: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:16:50.469526: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:16:50.470171: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:16:50.470336: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:16:50.470368: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:16:50.470385: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:16:50.470562: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:16:50.471243: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:16:50.471866: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-38773
I0725 12:16:50.473144 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-38773
INFO:tensorflow:Running local_init_op.
I0725 12:16:51.468627 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:16:51.611430 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:17:03.292212 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:17:03.294019 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:17:03.298054 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.15s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.308
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.656
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.264
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.137
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.505
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.758
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.391
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.397
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.220
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.569
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-12:17:04
I0725 12:17:04.648325 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:17:04
INFO:tensorflow:Saving dict for global step 38773: DetectionBoxes_Precision/mAP = 0.30844772, DetectionBoxes_Precision/mAP (large) = 0.75846547, DetectionBoxes_Precision/mAP (medium) = 0.50450826, DetectionBoxes_Precision/mAP (small) = 0.13690612, DetectionBoxes_Precision/mAP@.50IOU = 0.65594774, DetectionBoxes_Precision/mAP@.75IOU = 0.2635879, DetectionBoxes_Recall/AR@1 = 0.14670658, DetectionBoxes_Recall/AR@10 = 0.39053893, DetectionBoxes_Recall/AR@100 = 0.39736527, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5689655, DetectionBoxes_Recall/AR@100 (small) = 0.22023256, Loss/classification_loss = 3.1052973, Loss/localization_loss = 0.7042617, Loss/regularization_loss = 0.40045223, Loss/total_loss = 4.2100115, global_step = 38773, learning_rate = 0.003, loss = 4.2100115
I0725 12:17:04.648668 139696487446400 estimator.py:2049] Saving dict for global step 38773: DetectionBoxes_Precision/mAP = 0.30844772, DetectionBoxes_Precision/mAP (large) = 0.75846547, DetectionBoxes_Precision/mAP (medium) = 0.50450826, DetectionBoxes_Precision/mAP (small) = 0.13690612, DetectionBoxes_Precision/mAP@.50IOU = 0.65594774, DetectionBoxes_Precision/mAP@.75IOU = 0.2635879, DetectionBoxes_Recall/AR@1 = 0.14670658, DetectionBoxes_Recall/AR@10 = 0.39053893, DetectionBoxes_Recall/AR@100 = 0.39736527, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5689655, DetectionBoxes_Recall/AR@100 (small) = 0.22023256, Loss/classification_loss = 3.1052973, Loss/localization_loss = 0.7042617, Loss/regularization_loss = 0.40045223, Loss/total_loss = 4.2100115, global_step = 38773, learning_rate = 0.003, loss = 4.2100115
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 38773: training/model.ckpt-38773
I0725 12:17:04.653266 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 38773: training/model.ckpt-38773
INFO:tensorflow:global_step/sec: 1.35932
I0725 12:17:18.989205 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.35932
INFO:tensorflow:loss = 2.0038452, step = 38800 (73.566 sec)
I0725 12:17:18.990595 139696487446400 basic_session_run_hooks.py:260] loss = 2.0038452, step = 38800 (73.566 sec)
INFO:tensorflow:global_step/sec: 1.9744
I0725 12:18:09.637453 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9744
INFO:tensorflow:loss = 2.0581114, step = 38900 (50.648 sec)
I0725 12:18:09.638900 139696487446400 basic_session_run_hooks.py:260] loss = 2.0581114, step = 38900 (50.648 sec)
INFO:tensorflow:global_step/sec: 1.99051
I0725 12:18:59.875781 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99051
INFO:tensorflow:loss = 2.051322, step = 39000 (50.238 sec)
I0725 12:18:59.877326 139696487446400 basic_session_run_hooks.py:260] loss = 2.051322, step = 39000 (50.238 sec)
INFO:tensorflow:global_step/sec: 1.9838
I0725 12:19:50.284158 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9838
INFO:tensorflow:loss = 1.9616863, step = 39100 (50.408 sec)
I0725 12:19:50.285543 139696487446400 basic_session_run_hooks.py:260] loss = 1.9616863, step = 39100 (50.408 sec)
INFO:tensorflow:global_step/sec: 1.97694
I0725 12:20:40.867531 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97694
INFO:tensorflow:loss = 2.9596977, step = 39200 (50.583 sec)
I0725 12:20:40.868932 139696487446400 basic_session_run_hooks.py:260] loss = 2.9596977, step = 39200 (50.583 sec)
INFO:tensorflow:global_step/sec: 1.95868
I0725 12:21:31.922196 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95868
INFO:tensorflow:loss = 2.2324307, step = 39300 (51.055 sec)
I0725 12:21:31.923456 139696487446400 basic_session_run_hooks.py:260] loss = 2.2324307, step = 39300 (51.055 sec)
INFO:tensorflow:global_step/sec: 1.97192
I0725 12:22:22.634215 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97192
INFO:tensorflow:loss = 2.236891, step = 39400 (50.712 sec)
I0725 12:22:22.635430 139696487446400 basic_session_run_hooks.py:260] loss = 2.236891, step = 39400 (50.712 sec)
INFO:tensorflow:global_step/sec: 1.97243
I0725 12:23:13.333169 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97243
INFO:tensorflow:loss = 1.4197638, step = 39500 (50.699 sec)
I0725 12:23:13.334424 139696487446400 basic_session_run_hooks.py:260] loss = 1.4197638, step = 39500 (50.699 sec)
INFO:tensorflow:global_step/sec: 1.9579
I0725 12:24:04.408342 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9579
INFO:tensorflow:loss = 2.7245672, step = 39600 (51.075 sec)
I0725 12:24:04.409800 139696487446400 basic_session_run_hooks.py:260] loss = 2.7245672, step = 39600 (51.075 sec)
INFO:tensorflow:global_step/sec: 1.96147
I0725 12:24:55.390357 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96147
INFO:tensorflow:loss = 2.1953971, step = 39700 (50.982 sec)
I0725 12:24:55.391941 139696487446400 basic_session_run_hooks.py:260] loss = 2.1953971, step = 39700 (50.982 sec)
INFO:tensorflow:global_step/sec: 1.98925
I0725 12:25:45.660525 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98925
INFO:tensorflow:loss = 2.735447, step = 39800 (50.270 sec)
I0725 12:25:45.661906 139696487446400 basic_session_run_hooks.py:260] loss = 2.735447, step = 39800 (50.270 sec)
INFO:tensorflow:global_step/sec: 1.9915
I0725 12:26:35.874012 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9915
INFO:tensorflow:loss = 2.4304798, step = 39900 (50.213 sec)
I0725 12:26:35.875195 139696487446400 basic_session_run_hooks.py:260] loss = 2.4304798, step = 39900 (50.213 sec)
INFO:tensorflow:Saving checkpoints for 39914 into training/model.ckpt.
I0725 12:26:42.352771 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 39914 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37e85f8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:26:44.041338 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd37e85f8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a12f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:26:44.267119 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a12f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:26:45.342550 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.148882 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.189361 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.230729 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.271571 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.312286 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:26:48.352092 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:26:50.276139 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:26:50Z
I0725 12:26:50.298322 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:26:50Z
INFO:tensorflow:Graph was finalized.
I0725 12:26:50.837937 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:26:50.838746: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:26:50.839415: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:26:50.839588: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:26:50.839655: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:26:50.839702: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:26:50.839769: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:26:50.839844: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:26:50.839916: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:26:50.839966: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:26:50.840126: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:26:50.840858: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:26:50.841421: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:26:50.841485: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:26:50.841506: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:26:50.841523: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:26:50.841729: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:26:50.842411: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:26:50.843040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-39914
I0725 12:26:50.844715 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-39914
INFO:tensorflow:Running local_init_op.
I0725 12:26:51.873568 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:26:52.035385 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:27:03.354081 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:27:03.355198 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:27:03.358200 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.05s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.314
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.647
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.257
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.144
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.513
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.731
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.149
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.398
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.404
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.227
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.579
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.769
INFO:tensorflow:Finished evaluation at 2020-07-25-12:27:04
I0725 12:27:04.616760 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:27:04
INFO:tensorflow:Saving dict for global step 39914: DetectionBoxes_Precision/mAP = 0.31408024, DetectionBoxes_Precision/mAP (large) = 0.7305077, DetectionBoxes_Precision/mAP (medium) = 0.51342905, DetectionBoxes_Precision/mAP (small) = 0.14382808, DetectionBoxes_Precision/mAP@.50IOU = 0.6469415, DetectionBoxes_Precision/mAP@.75IOU = 0.2571124, DetectionBoxes_Recall/AR@1 = 0.14862275, DetectionBoxes_Recall/AR@10 = 0.39820358, DetectionBoxes_Recall/AR@100 = 0.4043114, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5787798, DetectionBoxes_Recall/AR@100 (small) = 0.2272093, Loss/classification_loss = 3.0709727, Loss/localization_loss = 0.6704679, Loss/regularization_loss = 0.39521497, Loss/total_loss = 4.1366563, global_step = 39914, learning_rate = 0.003, loss = 4.1366563
I0725 12:27:04.617133 139696487446400 estimator.py:2049] Saving dict for global step 39914: DetectionBoxes_Precision/mAP = 0.31408024, DetectionBoxes_Precision/mAP (large) = 0.7305077, DetectionBoxes_Precision/mAP (medium) = 0.51342905, DetectionBoxes_Precision/mAP (small) = 0.14382808, DetectionBoxes_Precision/mAP@.50IOU = 0.6469415, DetectionBoxes_Precision/mAP@.75IOU = 0.2571124, DetectionBoxes_Recall/AR@1 = 0.14862275, DetectionBoxes_Recall/AR@10 = 0.39820358, DetectionBoxes_Recall/AR@100 = 0.4043114, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.5787798, DetectionBoxes_Recall/AR@100 (small) = 0.2272093, Loss/classification_loss = 3.0709727, Loss/localization_loss = 0.6704679, Loss/regularization_loss = 0.39521497, Loss/total_loss = 4.1366563, global_step = 39914, learning_rate = 0.003, loss = 4.1366563
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 39914: training/model.ckpt-39914
I0725 12:27:04.621881 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 39914: training/model.ckpt-39914
INFO:tensorflow:global_step/sec: 1.37243
I0725 12:27:48.737200 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.37243
INFO:tensorflow:loss = 2.3399282, step = 40000 (72.863 sec)
I0725 12:27:48.738420 139696487446400 basic_session_run_hooks.py:260] loss = 2.3399282, step = 40000 (72.863 sec)
INFO:tensorflow:global_step/sec: 1.98004
I0725 12:28:39.241165 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98004
INFO:tensorflow:loss = 2.0097272, step = 40100 (50.504 sec)
I0725 12:28:39.242338 139696487446400 basic_session_run_hooks.py:260] loss = 2.0097272, step = 40100 (50.504 sec)
INFO:tensorflow:global_step/sec: 1.98907
I0725 12:29:29.515968 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98907
INFO:tensorflow:loss = 2.4238997, step = 40200 (50.275 sec)
I0725 12:29:29.517248 139696487446400 basic_session_run_hooks.py:260] loss = 2.4238997, step = 40200 (50.275 sec)
INFO:tensorflow:global_step/sec: 1.97899
I0725 12:30:20.046707 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97899
INFO:tensorflow:loss = 2.2074132, step = 40300 (50.531 sec)
I0725 12:30:20.047877 139696487446400 basic_session_run_hooks.py:260] loss = 2.2074132, step = 40300 (50.531 sec)
INFO:tensorflow:global_step/sec: 1.9898
I0725 12:31:10.302901 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9898
INFO:tensorflow:loss = 2.6471748, step = 40400 (50.256 sec)
I0725 12:31:10.304217 139696487446400 basic_session_run_hooks.py:260] loss = 2.6471748, step = 40400 (50.256 sec)
INFO:tensorflow:global_step/sec: 1.97868
I0725 12:32:00.841710 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97868
INFO:tensorflow:loss = 2.4614217, step = 40500 (50.539 sec)
I0725 12:32:00.843171 139696487446400 basic_session_run_hooks.py:260] loss = 2.4614217, step = 40500 (50.539 sec)
INFO:tensorflow:global_step/sec: 1.97372
I0725 12:32:51.507491 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97372
INFO:tensorflow:loss = 2.4660337, step = 40600 (50.665 sec)
I0725 12:32:51.508637 139696487446400 basic_session_run_hooks.py:260] loss = 2.4660337, step = 40600 (50.665 sec)
INFO:tensorflow:global_step/sec: 1.98493
I0725 12:33:41.887051 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98493
INFO:tensorflow:loss = 1.9023253, step = 40700 (50.380 sec)
I0725 12:33:41.888175 139696487446400 basic_session_run_hooks.py:260] loss = 1.9023253, step = 40700 (50.380 sec)
INFO:tensorflow:global_step/sec: 1.97641
I0725 12:34:32.483731 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97641
INFO:tensorflow:loss = 2.9176085, step = 40800 (50.597 sec)
I0725 12:34:32.485156 139696487446400 basic_session_run_hooks.py:260] loss = 2.9176085, step = 40800 (50.597 sec)
INFO:tensorflow:global_step/sec: 1.9804
I0725 12:35:22.978523 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9804
INFO:tensorflow:loss = 2.9262486, step = 40900 (50.495 sec)
I0725 12:35:22.980103 139696487446400 basic_session_run_hooks.py:260] loss = 2.9262486, step = 40900 (50.495 sec)
INFO:tensorflow:global_step/sec: 1.97651
I0725 12:36:13.572706 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97651
INFO:tensorflow:loss = 1.5338308, step = 41000 (50.594 sec)
I0725 12:36:13.574003 139696487446400 basic_session_run_hooks.py:260] loss = 1.5338308, step = 41000 (50.594 sec)
INFO:tensorflow:Saving checkpoints for 41059 into training/model.ckpt.
I0725 12:36:42.764217 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 41059 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a92ef0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:36:44.419740 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a92ef0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a88598> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:36:44.637543 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a88598> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:36:45.285150 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.012351 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.057913 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.099321 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.138737 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.178081 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:36:48.217374 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:36:50.356312 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:36:50Z
I0725 12:36:50.377341 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:36:50Z
INFO:tensorflow:Graph was finalized.
I0725 12:36:50.937734 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:36:50.938715: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:36:50.939317: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:36:50.939417: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:36:50.939482: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:36:50.939536: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:36:50.939591: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:36:50.939639: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:36:50.939689: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:36:50.939787: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:36:50.939997: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:36:50.940735: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:36:50.941383: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:36:50.941551: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:36:50.941577: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:36:50.941592: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:36:50.942137: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:36:50.942924: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:36:50.943587: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-41059
I0725 12:36:50.945306 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-41059
INFO:tensorflow:Running local_init_op.
I0725 12:36:51.902777 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:36:52.062722 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:37:03.274618 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:37:03.277894 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:37:03.281228 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.08s).
Accumulating evaluation results...
DONE (t=0.08s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.327
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.668
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.272
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.164
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.525
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.788
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.150
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.408
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.417
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.233
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.595
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.824
INFO:tensorflow:Finished evaluation at 2020-07-25-12:37:04
I0725 12:37:04.543524 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:37:04
INFO:tensorflow:Saving dict for global step 41059: DetectionBoxes_Precision/mAP = 0.32670867, DetectionBoxes_Precision/mAP (large) = 0.78821576, DetectionBoxes_Precision/mAP (medium) = 0.5250071, DetectionBoxes_Precision/mAP (small) = 0.16388626, DetectionBoxes_Precision/mAP@.50IOU = 0.6683484, DetectionBoxes_Precision/mAP@.75IOU = 0.2721942, DetectionBoxes_Recall/AR@1 = 0.15005988, DetectionBoxes_Recall/AR@10 = 0.40826347, DetectionBoxes_Recall/AR@100 = 0.41688624, DetectionBoxes_Recall/AR@100 (large) = 0.8241379, DetectionBoxes_Recall/AR@100 (medium) = 0.5954907, DetectionBoxes_Recall/AR@100 (small) = 0.23302326, Loss/classification_loss = 3.0707886, Loss/localization_loss = 0.6618819, Loss/regularization_loss = 0.39018014, Loss/total_loss = 4.1228514, global_step = 41059, learning_rate = 0.003, loss = 4.1228514
I0725 12:37:04.543927 139696487446400 estimator.py:2049] Saving dict for global step 41059: DetectionBoxes_Precision/mAP = 0.32670867, DetectionBoxes_Precision/mAP (large) = 0.78821576, DetectionBoxes_Precision/mAP (medium) = 0.5250071, DetectionBoxes_Precision/mAP (small) = 0.16388626, DetectionBoxes_Precision/mAP@.50IOU = 0.6683484, DetectionBoxes_Precision/mAP@.75IOU = 0.2721942, DetectionBoxes_Recall/AR@1 = 0.15005988, DetectionBoxes_Recall/AR@10 = 0.40826347, DetectionBoxes_Recall/AR@100 = 0.41688624, DetectionBoxes_Recall/AR@100 (large) = 0.8241379, DetectionBoxes_Recall/AR@100 (medium) = 0.5954907, DetectionBoxes_Recall/AR@100 (small) = 0.23302326, Loss/classification_loss = 3.0707886, Loss/localization_loss = 0.6618819, Loss/regularization_loss = 0.39018014, Loss/total_loss = 4.1228514, global_step = 41059, learning_rate = 0.003, loss = 4.1228514
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 41059: training/model.ckpt-41059
I0725 12:37:04.548522 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 41059: training/model.ckpt-41059
INFO:tensorflow:global_step/sec: 1.38475
I0725 12:37:25.787925 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38475
INFO:tensorflow:loss = 1.788405, step = 41100 (72.215 sec)
I0725 12:37:25.789342 139696487446400 basic_session_run_hooks.py:260] loss = 1.788405, step = 41100 (72.215 sec)
INFO:tensorflow:global_step/sec: 1.99184
I0725 12:38:15.992783 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99184
INFO:tensorflow:loss = 2.301392, step = 41200 (50.205 sec)
I0725 12:38:15.994250 139696487446400 basic_session_run_hooks.py:260] loss = 2.301392, step = 41200 (50.205 sec)
INFO:tensorflow:global_step/sec: 1.9749
I0725 12:39:06.628358 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9749
INFO:tensorflow:loss = 1.8179144, step = 41300 (50.635 sec)
I0725 12:39:06.629623 139696487446400 basic_session_run_hooks.py:260] loss = 1.8179144, step = 41300 (50.635 sec)
INFO:tensorflow:global_step/sec: 1.98806
I0725 12:39:56.928687 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98806
INFO:tensorflow:loss = 2.7247133, step = 41400 (50.301 sec)
I0725 12:39:56.930141 139696487446400 basic_session_run_hooks.py:260] loss = 2.7247133, step = 41400 (50.301 sec)
INFO:tensorflow:global_step/sec: 1.97612
I0725 12:40:47.532962 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97612
INFO:tensorflow:loss = 3.2410612, step = 41500 (50.604 sec)
I0725 12:40:47.534209 139696487446400 basic_session_run_hooks.py:260] loss = 3.2410612, step = 41500 (50.604 sec)
INFO:tensorflow:global_step/sec: 1.97257
I0725 12:41:38.228231 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97257
INFO:tensorflow:loss = 3.640157, step = 41600 (50.695 sec)
I0725 12:41:38.229374 139696487446400 basic_session_run_hooks.py:260] loss = 3.640157, step = 41600 (50.695 sec)
INFO:tensorflow:global_step/sec: 1.97936
I0725 12:42:28.749669 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97936
INFO:tensorflow:loss = 2.9493625, step = 41700 (50.522 sec)
I0725 12:42:28.751025 139696487446400 basic_session_run_hooks.py:260] loss = 2.9493625, step = 41700 (50.522 sec)
INFO:tensorflow:global_step/sec: 1.97591
I0725 12:43:19.359217 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97591
INFO:tensorflow:loss = 2.408829, step = 41800 (50.609 sec)
I0725 12:43:19.360528 139696487446400 basic_session_run_hooks.py:260] loss = 2.408829, step = 41800 (50.609 sec)
INFO:tensorflow:global_step/sec: 1.97017
I0725 12:44:10.116325 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97017
INFO:tensorflow:loss = 2.301969, step = 41900 (50.757 sec)
I0725 12:44:10.117490 139696487446400 basic_session_run_hooks.py:260] loss = 2.301969, step = 41900 (50.757 sec)
INFO:tensorflow:global_step/sec: 1.97475
I0725 12:45:00.755631 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97475
INFO:tensorflow:loss = 1.8167775, step = 42000 (50.640 sec)
I0725 12:45:00.757199 139696487446400 basic_session_run_hooks.py:260] loss = 1.8167775, step = 42000 (50.640 sec)
INFO:tensorflow:global_step/sec: 1.96733
I0725 12:45:51.585978 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96733
INFO:tensorflow:loss = 2.1655736, step = 42100 (50.830 sec)
I0725 12:45:51.587157 139696487446400 basic_session_run_hooks.py:260] loss = 2.1655736, step = 42100 (50.830 sec)
INFO:tensorflow:global_step/sec: 1.98102
I0725 12:46:42.064956 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98102
INFO:tensorflow:loss = 2.2403367, step = 42200 (50.479 sec)
I0725 12:46:42.066220 139696487446400 basic_session_run_hooks.py:260] loss = 2.2403367, step = 42200 (50.479 sec)
INFO:tensorflow:Saving checkpoints for 42203 into training/model.ckpt.
I0725 12:46:43.067724 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 42203 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd31468d0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:46:44.767126 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd31468d0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd353b1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:46:44.984685 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd353b1e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:46:45.638181 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.423156 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.461561 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.510285 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.549519 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.588550 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:46:48.627799 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:46:50.514147 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:46:50Z
I0725 12:46:50.536949 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:46:50Z
INFO:tensorflow:Graph was finalized.
I0725 12:46:51.088632 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:46:51.089463: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:46:51.090118: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:46:51.090271: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:46:51.090328: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:46:51.090394: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:46:51.090452: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:46:51.090504: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:46:51.090560: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:46:51.090659: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:46:51.090926: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:46:51.091576: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:46:51.092134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:46:51.092198: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:46:51.092224: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:46:51.092240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:46:51.092400: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:46:51.093152: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:46:51.093778: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-42203
I0725 12:46:51.095200 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-42203
INFO:tensorflow:Running local_init_op.
I0725 12:46:52.089901 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:46:52.237032 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:47:03.541975 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:47:03.544198 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:47:03.546322 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.05s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.322
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.668
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.264
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.144
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.517
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.780
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.146
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.399
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.405
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.209
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.598
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.814
INFO:tensorflow:Finished evaluation at 2020-07-25-12:47:04
I0725 12:47:04.815340 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:47:04
INFO:tensorflow:Saving dict for global step 42203: DetectionBoxes_Precision/mAP = 0.3223993, DetectionBoxes_Precision/mAP (large) = 0.7798062, DetectionBoxes_Precision/mAP (medium) = 0.51700115, DetectionBoxes_Precision/mAP (small) = 0.14402811, DetectionBoxes_Precision/mAP@.50IOU = 0.66814274, DetectionBoxes_Precision/mAP@.75IOU = 0.26443058, DetectionBoxes_Recall/AR@1 = 0.14646706, DetectionBoxes_Recall/AR@10 = 0.39868262, DetectionBoxes_Recall/AR@100 = 0.40491018, DetectionBoxes_Recall/AR@100 (large) = 0.8137931, DetectionBoxes_Recall/AR@100 (medium) = 0.597878, DetectionBoxes_Recall/AR@100 (small) = 0.20860465, Loss/classification_loss = 3.05583, Loss/localization_loss = 0.729329, Loss/regularization_loss = 0.3853447, Loss/total_loss = 4.1705027, global_step = 42203, learning_rate = 0.003, loss = 4.1705027
I0725 12:47:04.815646 139696487446400 estimator.py:2049] Saving dict for global step 42203: DetectionBoxes_Precision/mAP = 0.3223993, DetectionBoxes_Precision/mAP (large) = 0.7798062, DetectionBoxes_Precision/mAP (medium) = 0.51700115, DetectionBoxes_Precision/mAP (small) = 0.14402811, DetectionBoxes_Precision/mAP@.50IOU = 0.66814274, DetectionBoxes_Precision/mAP@.75IOU = 0.26443058, DetectionBoxes_Recall/AR@1 = 0.14646706, DetectionBoxes_Recall/AR@10 = 0.39868262, DetectionBoxes_Recall/AR@100 = 0.40491018, DetectionBoxes_Recall/AR@100 (large) = 0.8137931, DetectionBoxes_Recall/AR@100 (medium) = 0.597878, DetectionBoxes_Recall/AR@100 (small) = 0.20860465, Loss/classification_loss = 3.05583, Loss/localization_loss = 0.729329, Loss/regularization_loss = 0.3853447, Loss/total_loss = 4.1705027, global_step = 42203, learning_rate = 0.003, loss = 4.1705027
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 42203: training/model.ckpt-42203
I0725 12:47:04.820174 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 42203: training/model.ckpt-42203
INFO:tensorflow:global_step/sec: 1.38307
I0725 12:47:54.367920 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38307
INFO:tensorflow:loss = 2.5037134, step = 42300 (72.303 sec)
I0725 12:47:54.369183 139696487446400 basic_session_run_hooks.py:260] loss = 2.5037134, step = 42300 (72.303 sec)
INFO:tensorflow:global_step/sec: 1.98379
I0725 12:48:44.776389 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98379
INFO:tensorflow:loss = 1.8976147, step = 42400 (50.409 sec)
I0725 12:48:44.777842 139696487446400 basic_session_run_hooks.py:260] loss = 1.8976147, step = 42400 (50.409 sec)
INFO:tensorflow:global_step/sec: 1.97969
I0725 12:49:35.289178 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97969
INFO:tensorflow:loss = 2.0346947, step = 42500 (50.513 sec)
I0725 12:49:35.290340 139696487446400 basic_session_run_hooks.py:260] loss = 2.0346947, step = 42500 (50.513 sec)
INFO:tensorflow:global_step/sec: 1.97991
I0725 12:50:25.796739 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97991
INFO:tensorflow:loss = 2.6924584, step = 42600 (50.508 sec)
I0725 12:50:25.798110 139696487446400 basic_session_run_hooks.py:260] loss = 2.6924584, step = 42600 (50.508 sec)
INFO:tensorflow:global_step/sec: 1.98019
I0725 12:51:16.296805 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98019
INFO:tensorflow:loss = 2.5657275, step = 42700 (50.500 sec)
I0725 12:51:16.298271 139696487446400 basic_session_run_hooks.py:260] loss = 2.5657275, step = 42700 (50.500 sec)
INFO:tensorflow:global_step/sec: 1.9856
I0725 12:52:06.659451 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9856
INFO:tensorflow:loss = 2.2690394, step = 42800 (50.362 sec)
I0725 12:52:06.660587 139696487446400 basic_session_run_hooks.py:260] loss = 2.2690394, step = 42800 (50.362 sec)
INFO:tensorflow:global_step/sec: 1.96421
I0725 12:52:57.570503 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96421
INFO:tensorflow:loss = 3.087205, step = 42900 (50.912 sec)
I0725 12:52:57.572103 139696487446400 basic_session_run_hooks.py:260] loss = 3.087205, step = 42900 (50.912 sec)
INFO:tensorflow:global_step/sec: 1.98034
I0725 12:53:48.066920 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98034
INFO:tensorflow:loss = 2.2737093, step = 43000 (50.496 sec)
I0725 12:53:48.068147 139696487446400 basic_session_run_hooks.py:260] loss = 2.2737093, step = 43000 (50.496 sec)
INFO:tensorflow:global_step/sec: 1.98204
I0725 12:54:38.519873 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98204
INFO:tensorflow:loss = 2.4488277, step = 43100 (50.453 sec)
I0725 12:54:38.521017 139696487446400 basic_session_run_hooks.py:260] loss = 2.4488277, step = 43100 (50.453 sec)
INFO:tensorflow:global_step/sec: 1.9783
I0725 12:55:29.068387 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9783
INFO:tensorflow:loss = 2.2693398, step = 43200 (50.549 sec)
I0725 12:55:29.069749 139696487446400 basic_session_run_hooks.py:260] loss = 2.2693398, step = 43200 (50.549 sec)
INFO:tensorflow:global_step/sec: 1.96945
I0725 12:56:19.843951 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96945
INFO:tensorflow:loss = 1.9928473, step = 43300 (50.776 sec)
I0725 12:56:19.845337 139696487446400 basic_session_run_hooks.py:260] loss = 1.9928473, step = 43300 (50.776 sec)
INFO:tensorflow:Saving checkpoints for 43347 into training/model.ckpt.
I0725 12:56:43.137466 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 43347 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3c71860>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 12:56:44.797065 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3c71860>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3d5c268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 12:56:45.020230 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3d5c268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 12:56:46.039632 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.712408 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.751406 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.788968 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.827192 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.865386 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 12:56:48.905548 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 12:56:50.748959 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T12:56:50Z
I0725 12:56:50.769596 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T12:56:50Z
INFO:tensorflow:Graph was finalized.
I0725 12:56:51.298531 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 12:56:51.299334: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:56:51.299933: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 12:56:51.300033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 12:56:51.300096: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 12:56:51.300174: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 12:56:51.300223: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 12:56:51.300278: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 12:56:51.300325: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 12:56:51.300372: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 12:56:51.300514: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:56:51.301152: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:56:51.301634: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 12:56:51.301760: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 12:56:51.301795: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 12:56:51.301829: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 12:56:51.301994: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:56:51.302568: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 12:56:51.303173: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-43347
I0725 12:56:51.304429 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-43347
INFO:tensorflow:Running local_init_op.
I0725 12:56:52.299461 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 12:56:52.447049 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 12:57:03.774088 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 12:57:03.775645 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 12:57:03.779167 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.03s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.302
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.651
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.242
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.514
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.753
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.142
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.398
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.407
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.233
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.575
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-12:57:05
I0725 12:57:05.033341 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-12:57:05
INFO:tensorflow:Saving dict for global step 43347: DetectionBoxes_Precision/mAP = 0.3022842, DetectionBoxes_Precision/mAP (large) = 0.75254744, DetectionBoxes_Precision/mAP (medium) = 0.51408064, DetectionBoxes_Precision/mAP (small) = 0.14604993, DetectionBoxes_Precision/mAP@.50IOU = 0.6508758, DetectionBoxes_Precision/mAP@.75IOU = 0.24235833, DetectionBoxes_Recall/AR@1 = 0.14203593, DetectionBoxes_Recall/AR@10 = 0.39832336, DetectionBoxes_Recall/AR@100 = 0.40682635, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5745358, DetectionBoxes_Recall/AR@100 (small) = 0.23348837, Loss/classification_loss = 3.0987625, Loss/localization_loss = 0.6714554, Loss/regularization_loss = 0.3809104, Loss/total_loss = 4.151129, global_step = 43347, learning_rate = 0.003, loss = 4.151129
I0725 12:57:05.033657 139696487446400 estimator.py:2049] Saving dict for global step 43347: DetectionBoxes_Precision/mAP = 0.3022842, DetectionBoxes_Precision/mAP (large) = 0.75254744, DetectionBoxes_Precision/mAP (medium) = 0.51408064, DetectionBoxes_Precision/mAP (small) = 0.14604993, DetectionBoxes_Precision/mAP@.50IOU = 0.6508758, DetectionBoxes_Precision/mAP@.75IOU = 0.24235833, DetectionBoxes_Recall/AR@1 = 0.14203593, DetectionBoxes_Recall/AR@10 = 0.39832336, DetectionBoxes_Recall/AR@100 = 0.40682635, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.5745358, DetectionBoxes_Recall/AR@100 (small) = 0.23348837, Loss/classification_loss = 3.0987625, Loss/localization_loss = 0.6714554, Loss/regularization_loss = 0.3809104, Loss/total_loss = 4.151129, global_step = 43347, learning_rate = 0.003, loss = 4.151129
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 43347: training/model.ckpt-43347
I0725 12:57:05.037717 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 43347: training/model.ckpt-43347
INFO:tensorflow:global_step/sec: 1.37746
I0725 12:57:32.441097 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.37746
INFO:tensorflow:loss = 2.1092575, step = 43400 (72.597 sec)
I0725 12:57:32.442325 139696487446400 basic_session_run_hooks.py:260] loss = 2.1092575, step = 43400 (72.597 sec)
INFO:tensorflow:global_step/sec: 1.98147
I0725 12:58:22.908840 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98147
INFO:tensorflow:loss = 2.5206497, step = 43500 (50.468 sec)
I0725 12:58:22.910563 139696487446400 basic_session_run_hooks.py:260] loss = 2.5206497, step = 43500 (50.468 sec)
INFO:tensorflow:global_step/sec: 1.97982
I0725 12:59:13.418499 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97982
INFO:tensorflow:loss = 2.0180187, step = 43600 (50.509 sec)
I0725 12:59:13.419800 139696487446400 basic_session_run_hooks.py:260] loss = 2.0180187, step = 43600 (50.509 sec)
INFO:tensorflow:global_step/sec: 1.98844
I0725 13:00:03.709161 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98844
INFO:tensorflow:loss = 2.0518935, step = 43700 (50.291 sec)
I0725 13:00:03.710670 139696487446400 basic_session_run_hooks.py:260] loss = 2.0518935, step = 43700 (50.291 sec)
INFO:tensorflow:global_step/sec: 1.97464
I0725 13:00:54.351350 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97464
INFO:tensorflow:loss = 2.5685918, step = 43800 (50.642 sec)
I0725 13:00:54.352608 139696487446400 basic_session_run_hooks.py:260] loss = 2.5685918, step = 43800 (50.642 sec)
INFO:tensorflow:global_step/sec: 1.9886
I0725 13:01:44.638064 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9886
INFO:tensorflow:loss = 2.0381093, step = 43900 (50.287 sec)
I0725 13:01:44.639366 139696487446400 basic_session_run_hooks.py:260] loss = 2.0381093, step = 43900 (50.287 sec)
INFO:tensorflow:global_step/sec: 1.98818
I0725 13:02:34.935445 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98818
INFO:tensorflow:loss = 1.9995859, step = 44000 (50.297 sec)
I0725 13:02:34.936748 139696487446400 basic_session_run_hooks.py:260] loss = 1.9995859, step = 44000 (50.297 sec)
INFO:tensorflow:global_step/sec: 1.99938
I0725 13:03:24.950954 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99938
INFO:tensorflow:loss = 1.9007239, step = 44100 (50.015 sec)
I0725 13:03:24.952216 139696487446400 basic_session_run_hooks.py:260] loss = 1.9007239, step = 44100 (50.015 sec)
INFO:tensorflow:global_step/sec: 1.97777
I0725 13:04:15.513002 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97777
INFO:tensorflow:loss = 2.1765532, step = 44200 (50.562 sec)
I0725 13:04:15.514426 139696487446400 basic_session_run_hooks.py:260] loss = 2.1765532, step = 44200 (50.562 sec)
INFO:tensorflow:global_step/sec: 1.99123
I0725 13:05:05.733066 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99123
INFO:tensorflow:loss = 2.2814717, step = 44300 (50.220 sec)
I0725 13:05:05.734259 139696487446400 basic_session_run_hooks.py:260] loss = 2.2814717, step = 44300 (50.220 sec)
INFO:tensorflow:global_step/sec: 1.98481
I0725 13:05:56.115669 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98481
INFO:tensorflow:loss = 2.3107615, step = 44400 (50.383 sec)
I0725 13:05:56.117209 139696487446400 basic_session_run_hooks.py:260] loss = 2.3107615, step = 44400 (50.383 sec)
INFO:tensorflow:Saving checkpoints for 44494 into training/model.ckpt.
I0725 13:06:43.179098 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 44494 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3714da0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:06:44.899094 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3714da0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd36ed730> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:06:45.123227 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd36ed730> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:06:45.783346 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:48.532851 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:48.883852 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:48.926372 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:48.976462 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:49.015950 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:06:49.056021 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:06:50.937597 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:06:50Z
I0725 13:06:50.959963 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:06:50Z
INFO:tensorflow:Graph was finalized.
I0725 13:06:51.524802 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:06:51.525661: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:06:51.526306: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:06:51.526456: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:06:51.526511: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:06:51.526566: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:06:51.526628: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:06:51.526678: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:06:51.526726: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:06:51.526786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:06:51.526947: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:06:51.527610: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:06:51.528157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:06:51.528218: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:06:51.528245: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:06:51.528268: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:06:51.528442: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:06:51.529105: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:06:51.529649: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-44494
I0725 13:06:51.531105 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-44494
INFO:tensorflow:Running local_init_op.
I0725 13:06:52.558348 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:06:52.702177 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:07:04.060258 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:07:04.061618 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 13:07:04.063409 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.05s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.329
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.677
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.277
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.168
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.522
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.750
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.151
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.411
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.420
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.249
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.588
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.786
INFO:tensorflow:Finished evaluation at 2020-07-25-13:07:05
I0725 13:07:05.350678 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:07:05
INFO:tensorflow:Saving dict for global step 44494: DetectionBoxes_Precision/mAP = 0.32919037, DetectionBoxes_Precision/mAP (large) = 0.74973506, DetectionBoxes_Precision/mAP (medium) = 0.5223464, DetectionBoxes_Precision/mAP (small) = 0.1682299, DetectionBoxes_Precision/mAP@.50IOU = 0.6771799, DetectionBoxes_Precision/mAP@.75IOU = 0.27680433, DetectionBoxes_Recall/AR@1 = 0.15053892, DetectionBoxes_Recall/AR@10 = 0.4106587, DetectionBoxes_Recall/AR@100 = 0.42047903, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.5877984, DetectionBoxes_Recall/AR@100 (small) = 0.24930233, Loss/classification_loss = 3.0579853, Loss/localization_loss = 0.65965414, Loss/regularization_loss = 0.37694865, Loss/total_loss = 4.0945888, global_step = 44494, learning_rate = 0.003, loss = 4.0945888
I0725 13:07:05.351047 139696487446400 estimator.py:2049] Saving dict for global step 44494: DetectionBoxes_Precision/mAP = 0.32919037, DetectionBoxes_Precision/mAP (large) = 0.74973506, DetectionBoxes_Precision/mAP (medium) = 0.5223464, DetectionBoxes_Precision/mAP (small) = 0.1682299, DetectionBoxes_Precision/mAP@.50IOU = 0.6771799, DetectionBoxes_Precision/mAP@.75IOU = 0.27680433, DetectionBoxes_Recall/AR@1 = 0.15053892, DetectionBoxes_Recall/AR@10 = 0.4106587, DetectionBoxes_Recall/AR@100 = 0.42047903, DetectionBoxes_Recall/AR@100 (large) = 0.7862069, DetectionBoxes_Recall/AR@100 (medium) = 0.5877984, DetectionBoxes_Recall/AR@100 (small) = 0.24930233, Loss/classification_loss = 3.0579853, Loss/localization_loss = 0.65965414, Loss/regularization_loss = 0.37694865, Loss/total_loss = 4.0945888, global_step = 44494, learning_rate = 0.003, loss = 4.0945888
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 44494: training/model.ckpt-44494
I0725 13:07:05.355896 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 44494: training/model.ckpt-44494
INFO:tensorflow:global_step/sec: 1.3713
I0725 13:07:09.039028 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3713
INFO:tensorflow:loss = 1.824708, step = 44500 (72.923 sec)
I0725 13:07:09.040158 139696487446400 basic_session_run_hooks.py:260] loss = 1.824708, step = 44500 (72.923 sec)
INFO:tensorflow:global_step/sec: 1.97849
I0725 13:07:59.582657 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97849
INFO:tensorflow:loss = 1.8148668, step = 44600 (50.544 sec)
I0725 13:07:59.583839 139696487446400 basic_session_run_hooks.py:260] loss = 1.8148668, step = 44600 (50.544 sec)
INFO:tensorflow:global_step/sec: 1.98188
I0725 13:08:50.039786 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98188
INFO:tensorflow:loss = 2.3210177, step = 44700 (50.457 sec)
I0725 13:08:50.041182 139696487446400 basic_session_run_hooks.py:260] loss = 2.3210177, step = 44700 (50.457 sec)
INFO:tensorflow:global_step/sec: 1.97859
I0725 13:09:40.580969 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97859
INFO:tensorflow:loss = 1.7678843, step = 44800 (50.541 sec)
I0725 13:09:40.582492 139696487446400 basic_session_run_hooks.py:260] loss = 1.7678843, step = 44800 (50.541 sec)
INFO:tensorflow:global_step/sec: 1.97379
I0725 13:10:31.244976 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97379
INFO:tensorflow:loss = 1.3690085, step = 44900 (50.664 sec)
I0725 13:10:31.246157 139696487446400 basic_session_run_hooks.py:260] loss = 1.3690085, step = 44900 (50.664 sec)
INFO:tensorflow:global_step/sec: 1.9787
I0725 13:11:21.783096 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9787
INFO:tensorflow:loss = 2.0141716, step = 45000 (50.538 sec)
I0725 13:11:21.784343 139696487446400 basic_session_run_hooks.py:260] loss = 2.0141716, step = 45000 (50.538 sec)
INFO:tensorflow:global_step/sec: 1.99668
I0725 13:12:11.866166 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99668
INFO:tensorflow:loss = 2.4667623, step = 45100 (50.083 sec)
I0725 13:12:11.867615 139696487446400 basic_session_run_hooks.py:260] loss = 2.4667623, step = 45100 (50.083 sec)
INFO:tensorflow:global_step/sec: 1.9735
I0725 13:13:02.537448 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9735
INFO:tensorflow:loss = 1.8218901, step = 45200 (50.671 sec)
I0725 13:13:02.538523 139696487446400 basic_session_run_hooks.py:260] loss = 1.8218901, step = 45200 (50.671 sec)
INFO:tensorflow:global_step/sec: 1.98068
I0725 13:13:53.025168 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98068
INFO:tensorflow:loss = 1.9985648, step = 45300 (50.488 sec)
I0725 13:13:53.026485 139696487446400 basic_session_run_hooks.py:260] loss = 1.9985648, step = 45300 (50.488 sec)
INFO:tensorflow:global_step/sec: 1.98506
I0725 13:14:43.401398 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98506
INFO:tensorflow:loss = 2.5657692, step = 45400 (50.376 sec)
I0725 13:14:43.402570 139696487446400 basic_session_run_hooks.py:260] loss = 2.5657692, step = 45400 (50.376 sec)
INFO:tensorflow:global_step/sec: 1.9868
I0725 13:15:33.733721 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9868
INFO:tensorflow:loss = 1.9475193, step = 45500 (50.333 sec)
I0725 13:15:33.735175 139696487446400 basic_session_run_hooks.py:260] loss = 1.9475193, step = 45500 (50.333 sec)
INFO:tensorflow:global_step/sec: 1.99147
I0725 13:16:23.947735 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99147
INFO:tensorflow:loss = 2.7291887, step = 45600 (50.215 sec)
I0725 13:16:23.949961 139696487446400 basic_session_run_hooks.py:260] loss = 2.7291887, step = 45600 (50.215 sec)
INFO:tensorflow:Saving checkpoints for 45640 into training/model.ckpt.
I0725 13:16:43.561840 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 45640 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd05ad4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:16:45.240008 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd05ad4a8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd39311e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:16:45.453623 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd39311e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:16:46.094194 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:48.783250 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:48.842461 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:48.881408 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:48.921114 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:48.961528 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:16:49.001311 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:16:50.872032 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:16:50Z
I0725 13:16:50.894558 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:16:50Z
INFO:tensorflow:Graph was finalized.
I0725 13:16:51.454806 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:16:51.455629: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:16:51.456259: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:16:51.456357: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:16:51.456423: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:16:51.456479: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:16:51.456531: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:16:51.456609: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:16:51.456661: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:16:51.456742: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:16:51.456934: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:16:51.457596: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:16:51.458121: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:16:51.458260: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:16:51.458290: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:16:51.458306: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:16:51.458465: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:16:51.459095: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:16:51.459651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-45640
I0725 13:16:51.460958 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-45640
INFO:tensorflow:Running local_init_op.
I0725 13:16:52.418647 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:16:52.548598 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:17:03.728571 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:17:03.730323 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 13:17:03.733302 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.09s).
Accumulating evaluation results...
DONE (t=0.08s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.323
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.680
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.272
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.162
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.511
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.692
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.146
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.396
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.403
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.227
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.577
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.745
INFO:tensorflow:Finished evaluation at 2020-07-25-13:17:04
I0725 13:17:04.985176 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:17:04
INFO:tensorflow:Saving dict for global step 45640: DetectionBoxes_Precision/mAP = 0.32294893, DetectionBoxes_Precision/mAP (large) = 0.6916316, DetectionBoxes_Precision/mAP (medium) = 0.51063806, DetectionBoxes_Precision/mAP (small) = 0.16249023, DetectionBoxes_Precision/mAP@.50IOU = 0.68026876, DetectionBoxes_Precision/mAP@.75IOU = 0.2715693, DetectionBoxes_Recall/AR@1 = 0.1463473, DetectionBoxes_Recall/AR@10 = 0.39556885, DetectionBoxes_Recall/AR@100 = 0.40251496, DetectionBoxes_Recall/AR@100 (large) = 0.74482757, DetectionBoxes_Recall/AR@100 (medium) = 0.57665783, DetectionBoxes_Recall/AR@100 (small) = 0.22697674, Loss/classification_loss = 3.2705905, Loss/localization_loss = 0.69286346, Loss/regularization_loss = 0.3732784, Loss/total_loss = 4.336731, global_step = 45640, learning_rate = 0.003, loss = 4.336731
I0725 13:17:04.985479 139696487446400 estimator.py:2049] Saving dict for global step 45640: DetectionBoxes_Precision/mAP = 0.32294893, DetectionBoxes_Precision/mAP (large) = 0.6916316, DetectionBoxes_Precision/mAP (medium) = 0.51063806, DetectionBoxes_Precision/mAP (small) = 0.16249023, DetectionBoxes_Precision/mAP@.50IOU = 0.68026876, DetectionBoxes_Precision/mAP@.75IOU = 0.2715693, DetectionBoxes_Recall/AR@1 = 0.1463473, DetectionBoxes_Recall/AR@10 = 0.39556885, DetectionBoxes_Recall/AR@100 = 0.40251496, DetectionBoxes_Recall/AR@100 (large) = 0.74482757, DetectionBoxes_Recall/AR@100 (medium) = 0.57665783, DetectionBoxes_Recall/AR@100 (small) = 0.22697674, Loss/classification_loss = 3.2705905, Loss/localization_loss = 0.69286346, Loss/regularization_loss = 0.3732784, Loss/total_loss = 4.336731, global_step = 45640, learning_rate = 0.003, loss = 4.336731
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 45640: training/model.ckpt-45640
I0725 13:17:04.989516 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 45640: training/model.ckpt-45640
INFO:tensorflow:global_step/sec: 1.3902
I0725 13:17:35.879836 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3902
INFO:tensorflow:loss = 2.9178605, step = 45700 (71.931 sec)
I0725 13:17:35.881022 139696487446400 basic_session_run_hooks.py:260] loss = 2.9178605, step = 45700 (71.931 sec)
INFO:tensorflow:global_step/sec: 1.99012
I0725 13:18:26.128012 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99012
INFO:tensorflow:loss = 2.011766, step = 45800 (50.248 sec)
I0725 13:18:26.129285 139696487446400 basic_session_run_hooks.py:260] loss = 2.011766, step = 45800 (50.248 sec)
INFO:tensorflow:global_step/sec: 1.98288
I0725 13:19:16.559638 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98288
INFO:tensorflow:loss = 1.9734195, step = 45900 (50.432 sec)
I0725 13:19:16.560961 139696487446400 basic_session_run_hooks.py:260] loss = 1.9734195, step = 45900 (50.432 sec)
INFO:tensorflow:global_step/sec: 1.97834
I0725 13:20:07.107067 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97834
INFO:tensorflow:loss = 3.010165, step = 46000 (50.547 sec)
I0725 13:20:07.108335 139696487446400 basic_session_run_hooks.py:260] loss = 3.010165, step = 46000 (50.547 sec)
INFO:tensorflow:global_step/sec: 1.99164
I0725 13:20:57.317069 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99164
INFO:tensorflow:loss = 2.2495646, step = 46100 (50.210 sec)
I0725 13:20:57.318429 139696487446400 basic_session_run_hooks.py:260] loss = 2.2495646, step = 46100 (50.210 sec)
INFO:tensorflow:global_step/sec: 1.98572
I0725 13:21:47.676583 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98572
INFO:tensorflow:loss = 1.7813277, step = 46200 (50.361 sec)
I0725 13:21:47.679029 139696487446400 basic_session_run_hooks.py:260] loss = 1.7813277, step = 46200 (50.361 sec)
INFO:tensorflow:global_step/sec: 1.96522
I0725 13:22:38.561521 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96522
INFO:tensorflow:loss = 2.2675092, step = 46300 (50.884 sec)
I0725 13:22:38.562904 139696487446400 basic_session_run_hooks.py:260] loss = 2.2675092, step = 46300 (50.884 sec)
INFO:tensorflow:global_step/sec: 1.97764
I0725 13:23:29.126862 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97764
INFO:tensorflow:loss = 2.5706353, step = 46400 (50.565 sec)
I0725 13:23:29.128281 139696487446400 basic_session_run_hooks.py:260] loss = 2.5706353, step = 46400 (50.565 sec)
INFO:tensorflow:global_step/sec: 1.93831
I0725 13:24:20.718154 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93831
INFO:tensorflow:loss = 1.8617325, step = 46500 (51.591 sec)
I0725 13:24:20.719342 139696487446400 basic_session_run_hooks.py:260] loss = 1.8617325, step = 46500 (51.591 sec)
INFO:tensorflow:global_step/sec: 1.96363
I0725 13:25:11.644427 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96363
INFO:tensorflow:loss = 2.041388, step = 46600 (50.927 sec)
I0725 13:25:11.646768 139696487446400 basic_session_run_hooks.py:260] loss = 2.041388, step = 46600 (50.927 sec)
INFO:tensorflow:global_step/sec: 1.96313
I0725 13:26:02.583256 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96313
INFO:tensorflow:loss = 2.025878, step = 46700 (50.938 sec)
I0725 13:26:02.584634 139696487446400 basic_session_run_hooks.py:260] loss = 2.025878, step = 46700 (50.938 sec)
INFO:tensorflow:Saving checkpoints for 46783 into training/model.ckpt.
I0725 13:26:43.627936 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 46783 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a13a58>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:26:45.264208 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3a13a58>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd33790d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:26:45.502648 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd33790d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:26:46.541946 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.202533 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.242515 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.282149 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.323997 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.363210 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:26:49.401669 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:26:51.253411 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:26:51Z
I0725 13:26:51.274130 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:26:51Z
INFO:tensorflow:Graph was finalized.
I0725 13:26:51.839105 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:26:51.839944: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:26:51.840524: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:26:51.840684: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:26:51.840741: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:26:51.840801: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:26:51.840882: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:26:51.840937: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:26:51.840988: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:26:51.841039: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:26:51.841193: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:26:51.841856: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:26:51.842424: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:26:51.842487: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:26:51.842514: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:26:51.842530: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:26:51.842698: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:26:51.843372: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:26:51.843962: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-46783
I0725 13:26:51.845295 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-46783
INFO:tensorflow:Running local_init_op.
I0725 13:26:52.844772 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:26:52.979073 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:27:04.216906 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:27:04.218473 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 13:27:04.221141 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.04s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.334
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.675
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.278
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.170
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.520
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.731
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.149
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.413
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.422
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.245
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.597
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.769
INFO:tensorflow:Finished evaluation at 2020-07-25-13:27:05
I0725 13:27:05.465714 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:27:05
INFO:tensorflow:Saving dict for global step 46783: DetectionBoxes_Precision/mAP = 0.33398843, DetectionBoxes_Precision/mAP (large) = 0.73056704, DetectionBoxes_Precision/mAP (medium) = 0.52037036, DetectionBoxes_Precision/mAP (small) = 0.17017087, DetectionBoxes_Precision/mAP@.50IOU = 0.67475605, DetectionBoxes_Precision/mAP@.75IOU = 0.27753365, DetectionBoxes_Recall/AR@1 = 0.14946108, DetectionBoxes_Recall/AR@10 = 0.41257486, DetectionBoxes_Recall/AR@100 = 0.42191616, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.59681696, DetectionBoxes_Recall/AR@100 (small) = 0.24488372, Loss/classification_loss = 2.9935215, Loss/localization_loss = 0.6291783, Loss/regularization_loss = 0.36973318, Loss/total_loss = 3.9924345, global_step = 46783, learning_rate = 0.003, loss = 3.9924345
I0725 13:27:05.466084 139696487446400 estimator.py:2049] Saving dict for global step 46783: DetectionBoxes_Precision/mAP = 0.33398843, DetectionBoxes_Precision/mAP (large) = 0.73056704, DetectionBoxes_Precision/mAP (medium) = 0.52037036, DetectionBoxes_Precision/mAP (small) = 0.17017087, DetectionBoxes_Precision/mAP@.50IOU = 0.67475605, DetectionBoxes_Precision/mAP@.75IOU = 0.27753365, DetectionBoxes_Recall/AR@1 = 0.14946108, DetectionBoxes_Recall/AR@10 = 0.41257486, DetectionBoxes_Recall/AR@100 = 0.42191616, DetectionBoxes_Recall/AR@100 (large) = 0.76896554, DetectionBoxes_Recall/AR@100 (medium) = 0.59681696, DetectionBoxes_Recall/AR@100 (small) = 0.24488372, Loss/classification_loss = 2.9935215, Loss/localization_loss = 0.6291783, Loss/regularization_loss = 0.36973318, Loss/total_loss = 3.9924345, global_step = 46783, learning_rate = 0.003, loss = 3.9924345
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 46783: training/model.ckpt-46783
I0725 13:27:05.470041 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 46783: training/model.ckpt-46783
INFO:tensorflow:global_step/sec: 1.38633
I0725 13:27:14.716018 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.38633
INFO:tensorflow:loss = 1.934505, step = 46800 (72.133 sec)
I0725 13:27:14.717225 139696487446400 basic_session_run_hooks.py:260] loss = 1.934505, step = 46800 (72.133 sec)
INFO:tensorflow:global_step/sec: 1.99244
I0725 13:28:04.905642 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99244
INFO:tensorflow:loss = 2.0489345, step = 46900 (50.190 sec)
I0725 13:28:04.906956 139696487446400 basic_session_run_hooks.py:260] loss = 2.0489345, step = 46900 (50.190 sec)
INFO:tensorflow:global_step/sec: 1.96197
I0725 13:28:55.874763 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96197
INFO:tensorflow:loss = 2.4395854, step = 47000 (50.969 sec)
I0725 13:28:55.876225 139696487446400 basic_session_run_hooks.py:260] loss = 2.4395854, step = 47000 (50.969 sec)
INFO:tensorflow:global_step/sec: 1.94377
I0725 13:29:47.321163 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94377
INFO:tensorflow:loss = 2.3275917, step = 47100 (51.446 sec)
I0725 13:29:47.322517 139696487446400 basic_session_run_hooks.py:260] loss = 2.3275917, step = 47100 (51.446 sec)
INFO:tensorflow:global_step/sec: 1.96273
I0725 13:30:38.270613 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96273
INFO:tensorflow:loss = 2.7436266, step = 47200 (50.949 sec)
I0725 13:30:38.271940 139696487446400 basic_session_run_hooks.py:260] loss = 2.7436266, step = 47200 (50.949 sec)
INFO:tensorflow:global_step/sec: 1.95783
I0725 13:31:29.347615 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95783
INFO:tensorflow:loss = 2.2631812, step = 47300 (51.077 sec)
I0725 13:31:29.348756 139696487446400 basic_session_run_hooks.py:260] loss = 2.2631812, step = 47300 (51.077 sec)
INFO:tensorflow:global_step/sec: 1.9888
I0725 13:32:19.629276 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.9888
INFO:tensorflow:loss = 2.3794904, step = 47400 (50.282 sec)
I0725 13:32:19.630376 139696487446400 basic_session_run_hooks.py:260] loss = 2.3794904, step = 47400 (50.282 sec)
INFO:tensorflow:global_step/sec: 1.99522
I0725 13:33:09.749156 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.99522
INFO:tensorflow:loss = 1.7546896, step = 47500 (50.120 sec)
I0725 13:33:09.750283 139696487446400 basic_session_run_hooks.py:260] loss = 1.7546896, step = 47500 (50.120 sec)
INFO:tensorflow:global_step/sec: 2.00118
I0725 13:33:59.719726 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 2.00118
INFO:tensorflow:loss = 1.4212245, step = 47600 (49.971 sec)
I0725 13:33:59.721069 139696487446400 basic_session_run_hooks.py:260] loss = 1.4212245, step = 47600 (49.971 sec)
INFO:tensorflow:global_step/sec: 1.94733
I0725 13:34:51.072169 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94733
INFO:tensorflow:loss = 2.3544445, step = 47700 (51.353 sec)
I0725 13:34:51.074509 139696487446400 basic_session_run_hooks.py:260] loss = 2.3544445, step = 47700 (51.353 sec)
INFO:tensorflow:global_step/sec: 1.93571
I0725 13:35:42.732718 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93571
INFO:tensorflow:loss = 1.9917847, step = 47800 (51.660 sec)
I0725 13:35:42.734097 139696487446400 basic_session_run_hooks.py:260] loss = 1.9917847, step = 47800 (51.660 sec)
INFO:tensorflow:global_step/sec: 1.97408
I0725 13:36:33.389224 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97408
INFO:tensorflow:loss = 2.3588371, step = 47900 (50.656 sec)
I0725 13:36:33.390511 139696487446400 basic_session_run_hooks.py:260] loss = 2.3588371, step = 47900 (50.656 sec)
INFO:tensorflow:Saving checkpoints for 47922 into training/model.ckpt.
I0725 13:36:43.990436 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 47922 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd2c176d8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:36:45.666163 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd2c176d8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a3b950> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:36:45.888174 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3a3b950> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:36:46.556254 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.273895 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.312585 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.352139 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.392471 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.433440 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:36:49.472296 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:36:51.671762 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:36:51Z
I0725 13:36:51.693767 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:36:51Z
INFO:tensorflow:Graph was finalized.
I0725 13:36:52.258656 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:36:52.259586: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:36:52.260246: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:36:52.260362: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:36:52.260492: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:36:52.260561: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:36:52.260615: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:36:52.260665: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:36:52.260714: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:36:52.260764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:36:52.260954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:36:52.261584: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:36:52.262134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:36:52.262288: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:36:52.262319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:36:52.262336: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:36:52.262533: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:36:52.263236: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:36:52.263807: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-47922
I0725 13:36:52.265123 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-47922
INFO:tensorflow:Running local_init_op.
I0725 13:36:53.273904 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:36:53.419204 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:37:04.857651 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:37:04.858797 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.01s)
I0725 13:37:04.864771 139694396827392 coco_tools.py:138] DONE (t=0.01s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.05s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.305
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.674
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.224
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.156
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.498
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.764
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.143
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.399
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.405
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.233
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.571
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-13:37:06
I0725 13:37:06.110659 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:37:06
INFO:tensorflow:Saving dict for global step 47922: DetectionBoxes_Precision/mAP = 0.3048236, DetectionBoxes_Precision/mAP (large) = 0.7638835, DetectionBoxes_Precision/mAP (medium) = 0.49796414, DetectionBoxes_Precision/mAP (small) = 0.15564474, DetectionBoxes_Precision/mAP@.50IOU = 0.6736909, DetectionBoxes_Precision/mAP@.75IOU = 0.2238397, DetectionBoxes_Recall/AR@1 = 0.14287426, DetectionBoxes_Recall/AR@10 = 0.39856288, DetectionBoxes_Recall/AR@100 = 0.4045509, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.57055706, DetectionBoxes_Recall/AR@100 (small) = 0.23348837, Loss/classification_loss = 3.1187212, Loss/localization_loss = 0.68357086, Loss/regularization_loss = 0.36625236, Loss/total_loss = 4.1685433, global_step = 47922, learning_rate = 0.003, loss = 4.1685433
I0725 13:37:06.111012 139696487446400 estimator.py:2049] Saving dict for global step 47922: DetectionBoxes_Precision/mAP = 0.3048236, DetectionBoxes_Precision/mAP (large) = 0.7638835, DetectionBoxes_Precision/mAP (medium) = 0.49796414, DetectionBoxes_Precision/mAP (small) = 0.15564474, DetectionBoxes_Precision/mAP@.50IOU = 0.6736909, DetectionBoxes_Precision/mAP@.75IOU = 0.2238397, DetectionBoxes_Recall/AR@1 = 0.14287426, DetectionBoxes_Recall/AR@10 = 0.39856288, DetectionBoxes_Recall/AR@100 = 0.4045509, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.57055706, DetectionBoxes_Recall/AR@100 (small) = 0.23348837, Loss/classification_loss = 3.1187212, Loss/localization_loss = 0.68357086, Loss/regularization_loss = 0.36625236, Loss/total_loss = 4.1685433, global_step = 47922, learning_rate = 0.003, loss = 4.1685433
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 47922: training/model.ckpt-47922
I0725 13:37:06.115282 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 47922: training/model.ckpt-47922
INFO:tensorflow:global_step/sec: 1.37452
I0725 13:37:46.141854 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.37452
INFO:tensorflow:loss = 1.5615135, step = 48000 (72.752 sec)
I0725 13:37:46.142958 139696487446400 basic_session_run_hooks.py:260] loss = 1.5615135, step = 48000 (72.752 sec)
INFO:tensorflow:global_step/sec: 1.96829
I0725 13:38:36.947440 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96829
INFO:tensorflow:loss = 2.1219382, step = 48100 (50.806 sec)
I0725 13:38:36.948854 139696487446400 basic_session_run_hooks.py:260] loss = 2.1219382, step = 48100 (50.806 sec)
INFO:tensorflow:global_step/sec: 1.96933
I0725 13:39:27.726202 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96933
INFO:tensorflow:loss = 1.9785299, step = 48200 (50.778 sec)
I0725 13:39:27.727314 139696487446400 basic_session_run_hooks.py:260] loss = 1.9785299, step = 48200 (50.778 sec)
INFO:tensorflow:global_step/sec: 1.96173
I0725 13:40:18.701550 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96173
INFO:tensorflow:loss = 3.7724192, step = 48300 (50.976 sec)
I0725 13:40:18.702937 139696487446400 basic_session_run_hooks.py:260] loss = 3.7724192, step = 48300 (50.976 sec)
INFO:tensorflow:global_step/sec: 1.94276
I0725 13:41:10.174615 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94276
INFO:tensorflow:loss = 1.701015, step = 48400 (51.473 sec)
I0725 13:41:10.176106 139696487446400 basic_session_run_hooks.py:260] loss = 1.701015, step = 48400 (51.473 sec)
INFO:tensorflow:global_step/sec: 1.93706
I0725 13:42:01.799192 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93706
INFO:tensorflow:loss = 1.7419478, step = 48500 (51.624 sec)
I0725 13:42:01.800551 139696487446400 basic_session_run_hooks.py:260] loss = 1.7419478, step = 48500 (51.624 sec)
INFO:tensorflow:global_step/sec: 1.95346
I0725 13:42:52.990410 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95346
INFO:tensorflow:loss = 2.061284, step = 48600 (51.191 sec)
I0725 13:42:52.991838 139696487446400 basic_session_run_hooks.py:260] loss = 2.061284, step = 48600 (51.191 sec)
INFO:tensorflow:global_step/sec: 1.96134
I0725 13:43:43.976034 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96134
INFO:tensorflow:loss = 2.454265, step = 48700 (50.986 sec)
I0725 13:43:43.977406 139696487446400 basic_session_run_hooks.py:260] loss = 2.454265, step = 48700 (50.986 sec)
INFO:tensorflow:global_step/sec: 1.95142
I0725 13:44:35.220803 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95142
INFO:tensorflow:loss = 2.2380974, step = 48800 (51.245 sec)
I0725 13:44:35.222217 139696487446400 basic_session_run_hooks.py:260] loss = 2.2380974, step = 48800 (51.245 sec)
INFO:tensorflow:global_step/sec: 1.97353
I0725 13:45:25.891329 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97353
INFO:tensorflow:loss = 2.5915976, step = 48900 (50.670 sec)
I0725 13:45:25.892527 139696487446400 basic_session_run_hooks.py:260] loss = 2.5915976, step = 48900 (50.670 sec)
INFO:tensorflow:global_step/sec: 1.95517
I0725 13:46:17.037724 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95517
INFO:tensorflow:loss = 1.7467698, step = 49000 (51.146 sec)
I0725 13:46:17.038944 139696487446400 basic_session_run_hooks.py:260] loss = 1.7467698, step = 49000 (51.146 sec)
INFO:tensorflow:Saving checkpoints for 49054 into training/model.ckpt.
I0725 13:46:44.354341 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 49054 into training/model.ckpt.
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d21d92550>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:46:46.048636 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0d21d92550>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d36c150d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:46:46.274853 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0d36c150d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:46:46.907252 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.648935 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.689533 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.730488 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.772722 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.811963 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:46:49.852056 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:46:51.757529 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:46:51Z
I0725 13:46:51.779488 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:46:51Z
INFO:tensorflow:Graph was finalized.
I0725 13:46:52.362612 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:46:52.363426: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:46:52.364061: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:46:52.364222: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:46:52.364279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:46:52.364324: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:46:52.364382: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:46:52.364428: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:46:52.364493: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:46:52.364540: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:46:52.364779: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:46:52.365464: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:46:52.366048: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:46:52.366124: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:46:52.366144: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:46:52.366172: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:46:52.366378: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:46:52.367161: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:46:52.367764: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-49054
I0725 13:46:52.369122 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-49054
INFO:tensorflow:Running local_init_op.
I0725 13:46:53.408309 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:46:53.561653 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:47:05.292099 139694396827392 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:47:05.293503 139694396827392 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 13:47:05.295591 139694396827392 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.09s).
Accumulating evaluation results...
DONE (t=0.11s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.323
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.660
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.259
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.160
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.508
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.754
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.147
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.395
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.402
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.228
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.570
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.793
INFO:tensorflow:Finished evaluation at 2020-07-25-13:47:06
I0725 13:47:06.586084 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:47:06
INFO:tensorflow:Saving dict for global step 49054: DetectionBoxes_Precision/mAP = 0.3226502, DetectionBoxes_Precision/mAP (large) = 0.7536055, DetectionBoxes_Precision/mAP (medium) = 0.50840014, DetectionBoxes_Precision/mAP (small) = 0.15983553, DetectionBoxes_Precision/mAP@.50IOU = 0.65956813, DetectionBoxes_Precision/mAP@.75IOU = 0.258721, DetectionBoxes_Recall/AR@1 = 0.14742514, DetectionBoxes_Recall/AR@10 = 0.3954491, DetectionBoxes_Recall/AR@100 = 0.40203592, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.57029176, DetectionBoxes_Recall/AR@100 (small) = 0.22767442, Loss/classification_loss = 3.1997683, Loss/localization_loss = 0.69695204, Loss/regularization_loss = 0.36294565, Loss/total_loss = 4.2596645, global_step = 49054, learning_rate = 0.003, loss = 4.2596645
I0725 13:47:06.586417 139696487446400 estimator.py:2049] Saving dict for global step 49054: DetectionBoxes_Precision/mAP = 0.3226502, DetectionBoxes_Precision/mAP (large) = 0.7536055, DetectionBoxes_Precision/mAP (medium) = 0.50840014, DetectionBoxes_Precision/mAP (small) = 0.15983553, DetectionBoxes_Precision/mAP@.50IOU = 0.65956813, DetectionBoxes_Precision/mAP@.75IOU = 0.258721, DetectionBoxes_Recall/AR@1 = 0.14742514, DetectionBoxes_Recall/AR@10 = 0.3954491, DetectionBoxes_Recall/AR@100 = 0.40203592, DetectionBoxes_Recall/AR@100 (large) = 0.79310346, DetectionBoxes_Recall/AR@100 (medium) = 0.57029176, DetectionBoxes_Recall/AR@100 (small) = 0.22767442, Loss/classification_loss = 3.1997683, Loss/localization_loss = 0.69695204, Loss/regularization_loss = 0.36294565, Loss/total_loss = 4.2596645, global_step = 49054, learning_rate = 0.003, loss = 4.2596645
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 49054: training/model.ckpt-49054
I0725 13:47:06.590510 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 49054: training/model.ckpt-49054
INFO:tensorflow:global_step/sec: 1.3499
I0725 13:47:31.117002 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.3499
INFO:tensorflow:loss = 2.6093361, step = 49100 (74.079 sec)
I0725 13:47:31.118189 139696487446400 basic_session_run_hooks.py:260] loss = 2.6093361, step = 49100 (74.079 sec)
INFO:tensorflow:global_step/sec: 1.92756
I0725 13:48:22.995952 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.92756
INFO:tensorflow:loss = 1.9086363, step = 49200 (51.879 sec)
I0725 13:48:22.997395 139696487446400 basic_session_run_hooks.py:260] loss = 1.9086363, step = 49200 (51.879 sec)
INFO:tensorflow:global_step/sec: 1.94639
I0725 13:49:14.373059 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.94639
INFO:tensorflow:loss = 2.2346869, step = 49300 (51.384 sec)
I0725 13:49:14.381038 139696487446400 basic_session_run_hooks.py:260] loss = 2.2346869, step = 49300 (51.384 sec)
INFO:tensorflow:global_step/sec: 1.95945
I0725 13:50:05.407749 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.95945
INFO:tensorflow:loss = 3.2552743, step = 49400 (51.028 sec)
I0725 13:50:05.409108 139696487446400 basic_session_run_hooks.py:260] loss = 3.2552743, step = 49400 (51.028 sec)
INFO:tensorflow:global_step/sec: 1.97073
I0725 13:50:56.150497 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.97073
INFO:tensorflow:loss = 2.373961, step = 49500 (50.743 sec)
I0725 13:50:56.151969 139696487446400 basic_session_run_hooks.py:260] loss = 2.373961, step = 49500 (50.743 sec)
INFO:tensorflow:global_step/sec: 1.98908
I0725 13:51:46.425005 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98908
INFO:tensorflow:loss = 2.4412563, step = 49600 (50.274 sec)
I0725 13:51:46.426387 139696487446400 basic_session_run_hooks.py:260] loss = 2.4412563, step = 49600 (50.274 sec)
INFO:tensorflow:global_step/sec: 1.98003
I0725 13:52:36.929313 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.98003
INFO:tensorflow:loss = 1.9997004, step = 49700 (50.504 sec)
I0725 13:52:36.930806 139696487446400 basic_session_run_hooks.py:260] loss = 1.9997004, step = 49700 (50.504 sec)
INFO:tensorflow:global_step/sec: 1.93652
I0725 13:53:28.568173 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.93652
INFO:tensorflow:loss = 1.5556808, step = 49800 (51.639 sec)
I0725 13:53:28.569524 139696487446400 basic_session_run_hooks.py:260] loss = 1.5556808, step = 49800 (51.639 sec)
INFO:tensorflow:global_step/sec: 1.96314
I0725 13:54:19.506968 139696487446400 basic_session_run_hooks.py:692] global_step/sec: 1.96314
INFO:tensorflow:loss = 2.3313603, step = 49900 (50.939 sec)
I0725 13:54:19.508162 139696487446400 basic_session_run_hooks.py:260] loss = 2.3313603, step = 49900 (50.939 sec)
INFO:tensorflow:Saving checkpoints for 50000 into training/model.ckpt.
I0725 13:55:10.190618 139696487446400 basic_session_run_hooks.py:606] Saving checkpoints for 50000 into training/model.ckpt.
INFO:tensorflow:Skip the current checkpoint eval due to throttle secs (600 secs).
I0725 13:55:11.770056 139696487446400 training.py:527] Skip the current checkpoint eval due to throttle secs (600 secs).
WARNING:tensorflow:Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3db8470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
W0725 13:55:11.841480 139696487446400 ag_logging.py:146] Entity <bound method TfExampleDecoder.decode of <object_detection.data_decoders.tf_example_decoder.TfExampleDecoder object at 0x7f0cd3db8470>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'
WARNING:tensorflow:Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3f87950> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
W0725 13:55:12.072944 139696487446400 ag_logging.py:146] Entity <function eval_input.<locals>.transform_and_pad_input_data_fn at 0x7f0cd3f87950> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4
INFO:tensorflow:Calling model_fn.
I0725 13:55:12.722950 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:15.829473 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:15.869073 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:15.908707 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:15.949070 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:15.989630 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:16.030720 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:55:17.867759 139696487446400 estimator.py:1150] Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2020-07-25T13:55:17Z
I0725 13:55:17.888903 139696487446400 evaluation.py:255] Starting evaluation at 2020-07-25T13:55:17Z
INFO:tensorflow:Graph was finalized.
I0725 13:55:18.433020 139696487446400 monitored_session.py:240] Graph was finalized.
2020-07-25 13:55:18.433893: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:18.434442: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:55:18.434532: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:55:18.434613: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:55:18.434666: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:55:18.434722: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:55:18.434770: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:55:18.434840: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:55:18.434890: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:55:18.435032: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:18.435681: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:18.436208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:55:18.436375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:55:18.436403: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:55:18.436419: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:55:18.436592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:18.437196: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:18.437726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-50000
I0725 13:55:18.439064 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-50000
INFO:tensorflow:Running local_init_op.
I0725 13:55:19.388922 139696487446400 session_manager.py:500] Running local_init_op.
INFO:tensorflow:Done running local_init_op.
I0725 13:55:19.525641 139696487446400 session_manager.py:502] Done running local_init_op.
INFO:tensorflow:Performing evaluation on 200 images.
I0725 13:55:30.802596 139694405220096 coco_evaluation.py:237] Performing evaluation on 200 images.
creating index...
index created!
INFO:tensorflow:Loading and preparing annotation results...
I0725 13:55:30.803688 139694405220096 coco_tools.py:116] Loading and preparing annotation results...
INFO:tensorflow:DONE (t=0.00s)
I0725 13:55:30.805570 139694405220096 coco_tools.py:138] DONE (t=0.00s)
creating index...
index created!
Running per image evaluation...
Evaluate annotation type *bbox*
DONE (t=1.04s).
Accumulating evaluation results...
DONE (t=0.10s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.338
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.682
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.304
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.172
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.534
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.717
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.149
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.413
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.420
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.241
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.600
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.741
INFO:tensorflow:Finished evaluation at 2020-07-25-13:55:32
I0725 13:55:32.081800 139696487446400 evaluation.py:275] Finished evaluation at 2020-07-25-13:55:32
INFO:tensorflow:Saving dict for global step 50000: DetectionBoxes_Precision/mAP = 0.33817786, DetectionBoxes_Precision/mAP (large) = 0.7171105, DetectionBoxes_Precision/mAP (medium) = 0.53408295, DetectionBoxes_Precision/mAP (small) = 0.17210937, DetectionBoxes_Precision/mAP@.50IOU = 0.6818491, DetectionBoxes_Precision/mAP@.75IOU = 0.30421385, DetectionBoxes_Recall/AR@1 = 0.14886227, DetectionBoxes_Recall/AR@10 = 0.41329342, DetectionBoxes_Recall/AR@100 = 0.4202395, DetectionBoxes_Recall/AR@100 (large) = 0.7413793, DetectionBoxes_Recall/AR@100 (medium) = 0.60026526, DetectionBoxes_Recall/AR@100 (small) = 0.24116279, Loss/classification_loss = 3.1889238, Loss/localization_loss = 0.62982416, Loss/regularization_loss = 0.36049584, Loss/total_loss = 4.1792436, global_step = 50000, learning_rate = 0.003, loss = 4.1792436
I0725 13:55:32.082226 139696487446400 estimator.py:2049] Saving dict for global step 50000: DetectionBoxes_Precision/mAP = 0.33817786, DetectionBoxes_Precision/mAP (large) = 0.7171105, DetectionBoxes_Precision/mAP (medium) = 0.53408295, DetectionBoxes_Precision/mAP (small) = 0.17210937, DetectionBoxes_Precision/mAP@.50IOU = 0.6818491, DetectionBoxes_Precision/mAP@.75IOU = 0.30421385, DetectionBoxes_Recall/AR@1 = 0.14886227, DetectionBoxes_Recall/AR@10 = 0.41329342, DetectionBoxes_Recall/AR@100 = 0.4202395, DetectionBoxes_Recall/AR@100 (large) = 0.7413793, DetectionBoxes_Recall/AR@100 (medium) = 0.60026526, DetectionBoxes_Recall/AR@100 (small) = 0.24116279, Loss/classification_loss = 3.1889238, Loss/localization_loss = 0.62982416, Loss/regularization_loss = 0.36049584, Loss/total_loss = 4.1792436, global_step = 50000, learning_rate = 0.003, loss = 4.1792436
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 50000: training/model.ckpt-50000
I0725 13:55:32.086631 139696487446400 estimator.py:2109] Saving 'checkpoint_path' summary for global step 50000: training/model.ckpt-50000
INFO:tensorflow:Performing the final export in the end of training.
I0725 13:55:32.087445 139696487446400 exporter.py:410] Performing the final export in the end of training.
INFO:tensorflow:Calling model_fn.
I0725 13:55:32.416783 139696487446400 estimator.py:1148] Calling model_fn.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.107863 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.147173 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.185689 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.226615 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.274488 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:55:35.314176 139696487446400 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:Done calling model_fn.
I0725 13:55:36.146296 139696487446400 estimator.py:1150] Done calling model_fn.
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
W0725 13:55:36.146745 139696487446400 deprecation.py:323] From /tensorflow-1.15.2/python3.6/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
INFO:tensorflow:Signatures INCLUDED in export for Classify: None
I0725 13:55:36.147641 139696487446400 export_utils.py:170] Signatures INCLUDED in export for Classify: None
INFO:tensorflow:Signatures INCLUDED in export for Regress: None
I0725 13:55:36.147793 139696487446400 export_utils.py:170] Signatures INCLUDED in export for Regress: None
INFO:tensorflow:Signatures INCLUDED in export for Predict: ['tensorflow/serving/predict', 'serving_default']
I0725 13:55:36.147963 139696487446400 export_utils.py:170] Signatures INCLUDED in export for Predict: ['tensorflow/serving/predict', 'serving_default']
INFO:tensorflow:Signatures INCLUDED in export for Train: None
I0725 13:55:36.148090 139696487446400 export_utils.py:170] Signatures INCLUDED in export for Train: None
INFO:tensorflow:Signatures INCLUDED in export for Eval: None
I0725 13:55:36.148219 139696487446400 export_utils.py:170] Signatures INCLUDED in export for Eval: None
2020-07-25 13:55:36.148984: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:36.149567: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:55:36.149666: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:55:36.149731: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:55:36.149786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:55:36.149855: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:55:36.149908: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:55:36.149960: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:55:36.150025: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:55:36.150182: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:36.150886: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:36.151431: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:55:36.151487: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:55:36.151514: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:55:36.151530: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:55:36.151702: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:36.152359: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:55:36.152936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-50000
I0725 13:55:36.155870 139696487446400 saver.py:1284] Restoring parameters from training/model.ckpt-50000
INFO:tensorflow:Assets added to graph.
I0725 13:55:36.621932 139696487446400 builder_impl.py:665] Assets added to graph.
INFO:tensorflow:No assets to write.
I0725 13:55:36.622152 139696487446400 builder_impl.py:460] No assets to write.
INFO:tensorflow:SavedModel written to: training/export/Servo/temp-b'1595685332'/saved_model.pb
I0725 13:55:37.462962 139696487446400 builder_impl.py:425] SavedModel written to: training/export/Servo/temp-b'1595685332'/saved_model.pb
INFO:tensorflow:Loss for final step: 1.8403296.
I0725 13:55:37.839620 139696487446400 estimator.py:371] Loss for final step: 1.8403296.
exporting the model¶
In [87]:
#the location where the exported model will be saved in.
output_directory = '/content/aeroplane_detection/models/research/fine_tuned_model'
# goes through the model is the training/ dir and gets the last one.
# you could choose a specfic one instead of the last
lst = os.listdir(model_dir)
lst = [l for l in lst if 'model.ckpt-' in l and '.meta' in l]
steps=np.array([int(re.findall('\d+', l)[0]) for l in lst])
last_model = lst[steps.argmax()].replace('.meta', '')
last_model_path = os.path.join(model_dir, last_model)
print(last_model_path)
#exports the model specifed and inference graph
!python /content/aeroplane_detection/models/research/object_detection/export_inference_graph.py \
--input_type=image_tensor \
--pipeline_config_path={model_pipline} \
--output_directory={output_directory} \
--trained_checkpoint_prefix={last_model_path}
training/model.ckpt-50000
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tf_slim/layers/layers.py:1089: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
W0725 13:58:37.323210 139719421781888 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tf_slim/layers/layers.py:1089: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.014420 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.067008 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.228652 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.286538 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.336991 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
INFO:tensorflow:depth of additional conv before box predictor: 0
I0725 13:58:40.386199 139719421781888 convolutional_box_predictor.py:156] depth of additional conv before box predictor: 0
WARNING:tensorflow:From /content/aeroplane_detection/models/research/object_detection/core/post_processing.py:583: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
W0725 13:58:40.723052 139719421781888 deprecation.py:323] From /content/aeroplane_detection/models/research/object_detection/core/post_processing.py:583: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From /content/aeroplane_detection/models/research/object_detection/exporter.py:474: get_or_create_global_step (from tf_slim.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Please switch to tf.train.get_or_create_global_step
W0725 13:58:41.125655 139719421781888 deprecation.py:323] From /content/aeroplane_detection/models/research/object_detection/exporter.py:474: get_or_create_global_step (from tf_slim.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Please switch to tf.train.get_or_create_global_step
WARNING:tensorflow:From /content/aeroplane_detection/models/research/object_detection/exporter.py:653: print_model_analysis (from tensorflow.contrib.tfprof.model_analyzer) is deprecated and will be removed after 2018-01-01.
Instructions for updating:
Use `tf.profiler.profile(graph, run_meta, op_log, cmd, options)`. Build `options` with `tf.profiler.ProfileOptionBuilder`. See README.md for details
W0725 13:58:41.130242 139719421781888 deprecation.py:323] From /content/aeroplane_detection/models/research/object_detection/exporter.py:653: print_model_analysis (from tensorflow.contrib.tfprof.model_analyzer) is deprecated and will be removed after 2018-01-01.
Instructions for updating:
Use `tf.profiler.profile(graph, run_meta, op_log, cmd, options)`. Build `options` with `tf.profiler.ProfileOptionBuilder`. See README.md for details
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/profiler/internal/flops_registry.py:142: tensor_shape_from_node_def_name (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.tensor_shape_from_node_def_name`
W0725 13:58:41.130941 139719421781888 deprecation.py:323] From /tensorflow-1.15.2/python3.6/tensorflow_core/python/profiler/internal/flops_registry.py:142: tensor_shape_from_node_def_name (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.tensor_shape_from_node_def_name`
133 ops no flops stats due to incomplete shapes.
Parsing Inputs...
Incomplete shape.
=========================Options=============================
-max_depth 10000
-min_bytes 0
-min_peak_bytes 0
-min_residual_bytes 0
-min_output_bytes 0
-min_micros 0
-min_accelerator_micros 0
-min_cpu_micros 0
-min_params 0
-min_float_ops 0
-min_occurrence 0
-step -1
-order_by name
-account_type_regexes _trainable_variables
-start_name_regexes .*
-trim_name_regexes .*BatchNorm.*
-show_name_regexes .*
-hide_name_regexes
-account_displayed_op_only true
-select params
-output stdout:
==================Model Analysis Report======================
Incomplete shape.
Doc:
scope: The nodes in the model graph are organized by their names, which is hierarchical like filesystem.
param: Number of parameters (in the Variable).
Profile:
node name | # parameters
_TFProfRoot (--/4.57m params)
BoxPredictor_0 (--/10.39k params)
BoxPredictor_0/BoxEncodingPredictor (--/6.92k params)
BoxPredictor_0/BoxEncodingPredictor/biases (12, 12/12 params)
BoxPredictor_0/BoxEncodingPredictor/weights (1x1x576x12, 6.91k/6.91k params)
BoxPredictor_0/ClassPredictor (--/3.46k params)
BoxPredictor_0/ClassPredictor/biases (6, 6/6 params)
BoxPredictor_0/ClassPredictor/weights (1x1x576x6, 3.46k/3.46k params)
BoxPredictor_1 (--/46.12k params)
BoxPredictor_1/BoxEncodingPredictor (--/30.74k params)
BoxPredictor_1/BoxEncodingPredictor/biases (24, 24/24 params)
BoxPredictor_1/BoxEncodingPredictor/weights (1x1x1280x24, 30.72k/30.72k params)
BoxPredictor_1/ClassPredictor (--/15.37k params)
BoxPredictor_1/ClassPredictor/biases (12, 12/12 params)
BoxPredictor_1/ClassPredictor/weights (1x1x1280x12, 15.36k/15.36k params)
BoxPredictor_2 (--/18.47k params)
BoxPredictor_2/BoxEncodingPredictor (--/12.31k params)
BoxPredictor_2/BoxEncodingPredictor/biases (24, 24/24 params)
BoxPredictor_2/BoxEncodingPredictor/weights (1x1x512x24, 12.29k/12.29k params)
BoxPredictor_2/ClassPredictor (--/6.16k params)
BoxPredictor_2/ClassPredictor/biases (12, 12/12 params)
BoxPredictor_2/ClassPredictor/weights (1x1x512x12, 6.14k/6.14k params)
BoxPredictor_3 (--/9.25k params)
BoxPredictor_3/BoxEncodingPredictor (--/6.17k params)
BoxPredictor_3/BoxEncodingPredictor/biases (24, 24/24 params)
BoxPredictor_3/BoxEncodingPredictor/weights (1x1x256x24, 6.14k/6.14k params)
BoxPredictor_3/ClassPredictor (--/3.08k params)
BoxPredictor_3/ClassPredictor/biases (12, 12/12 params)
BoxPredictor_3/ClassPredictor/weights (1x1x256x12, 3.07k/3.07k params)
BoxPredictor_4 (--/9.25k params)
BoxPredictor_4/BoxEncodingPredictor (--/6.17k params)
BoxPredictor_4/BoxEncodingPredictor/biases (24, 24/24 params)
BoxPredictor_4/BoxEncodingPredictor/weights (1x1x256x24, 6.14k/6.14k params)
BoxPredictor_4/ClassPredictor (--/3.08k params)
BoxPredictor_4/ClassPredictor/biases (12, 12/12 params)
BoxPredictor_4/ClassPredictor/weights (1x1x256x12, 3.07k/3.07k params)
BoxPredictor_5 (--/4.64k params)
BoxPredictor_5/BoxEncodingPredictor (--/3.10k params)
BoxPredictor_5/BoxEncodingPredictor/biases (24, 24/24 params)
BoxPredictor_5/BoxEncodingPredictor/weights (1x1x128x24, 3.07k/3.07k params)
BoxPredictor_5/ClassPredictor (--/1.55k params)
BoxPredictor_5/ClassPredictor/biases (12, 12/12 params)
BoxPredictor_5/ClassPredictor/weights (1x1x128x12, 1.54k/1.54k params)
FeatureExtractor (--/4.48m params)
FeatureExtractor/MobilenetV2 (--/4.48m params)
FeatureExtractor/MobilenetV2/Conv (--/864 params)
FeatureExtractor/MobilenetV2/Conv/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/Conv/weights (3x3x3x32, 864/864 params)
FeatureExtractor/MobilenetV2/Conv_1 (--/409.60k params)
FeatureExtractor/MobilenetV2/Conv_1/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/Conv_1/weights (1x1x320x1280, 409.60k/409.60k params)
FeatureExtractor/MobilenetV2/expanded_conv (--/800 params)
FeatureExtractor/MobilenetV2/expanded_conv/depthwise (--/288 params)
FeatureExtractor/MobilenetV2/expanded_conv/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv/depthwise/depthwise_weights (3x3x32x1, 288/288 params)
FeatureExtractor/MobilenetV2/expanded_conv/project (--/512 params)
FeatureExtractor/MobilenetV2/expanded_conv/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv/project/weights (1x1x32x16, 512/512 params)
FeatureExtractor/MobilenetV2/expanded_conv_1 (--/4.70k params)
FeatureExtractor/MobilenetV2/expanded_conv_1/depthwise (--/864 params)
FeatureExtractor/MobilenetV2/expanded_conv_1/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_1/depthwise/depthwise_weights (3x3x96x1, 864/864 params)
FeatureExtractor/MobilenetV2/expanded_conv_1/expand (--/1.54k params)
FeatureExtractor/MobilenetV2/expanded_conv_1/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_1/expand/weights (1x1x16x96, 1.54k/1.54k params)
FeatureExtractor/MobilenetV2/expanded_conv_1/project (--/2.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_1/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_1/project/weights (1x1x96x24, 2.30k/2.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_10 (--/64.90k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/depthwise (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_10/depthwise/depthwise_weights (3x3x384x1, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/expand (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_10/expand/weights (1x1x64x384, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/project (--/36.86k params)
FeatureExtractor/MobilenetV2/expanded_conv_10/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_10/project/weights (1x1x384x96, 36.86k/36.86k params)
FeatureExtractor/MobilenetV2/expanded_conv_11 (--/115.78k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/depthwise (--/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_11/depthwise/depthwise_weights (3x3x576x1, 5.18k/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/expand (--/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_11/expand/weights (1x1x96x576, 55.30k/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/project (--/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_11/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_11/project/weights (1x1x576x96, 55.30k/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_12 (--/115.78k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/depthwise (--/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_12/depthwise/depthwise_weights (3x3x576x1, 5.18k/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/expand (--/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_12/expand/weights (1x1x96x576, 55.30k/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/project (--/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_12/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_12/project/weights (1x1x576x96, 55.30k/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_13 (--/152.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/depthwise (--/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_13/depthwise/depthwise_weights (3x3x576x1, 5.18k/5.18k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/expand (--/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_13/expand/weights (1x1x96x576, 55.30k/55.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/project (--/92.16k params)
FeatureExtractor/MobilenetV2/expanded_conv_13/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_13/project/weights (1x1x576x160, 92.16k/92.16k params)
FeatureExtractor/MobilenetV2/expanded_conv_14 (--/315.84k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/depthwise (--/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_14/depthwise/depthwise_weights (3x3x960x1, 8.64k/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/expand (--/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_14/expand/weights (1x1x160x960, 153.60k/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/project (--/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_14/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_14/project/weights (1x1x960x160, 153.60k/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_15 (--/315.84k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/depthwise (--/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_15/depthwise/depthwise_weights (3x3x960x1, 8.64k/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/expand (--/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_15/expand/weights (1x1x160x960, 153.60k/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/project (--/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_15/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_15/project/weights (1x1x960x160, 153.60k/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_16 (--/469.44k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/depthwise (--/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_16/depthwise/depthwise_weights (3x3x960x1, 8.64k/8.64k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/expand (--/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_16/expand/weights (1x1x160x960, 153.60k/153.60k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/project (--/307.20k params)
FeatureExtractor/MobilenetV2/expanded_conv_16/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_16/project/weights (1x1x960x320, 307.20k/307.20k params)
FeatureExtractor/MobilenetV2/expanded_conv_2 (--/8.21k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/depthwise (--/1.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_2/depthwise/depthwise_weights (3x3x144x1, 1.30k/1.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/expand (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_2/expand/weights (1x1x24x144, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/project (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_2/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_2/project/weights (1x1x144x24, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_3 (--/9.36k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/depthwise (--/1.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_3/depthwise/depthwise_weights (3x3x144x1, 1.30k/1.30k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/expand (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_3/expand/weights (1x1x24x144, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/project (--/4.61k params)
FeatureExtractor/MobilenetV2/expanded_conv_3/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_3/project/weights (1x1x144x32, 4.61k/4.61k params)
FeatureExtractor/MobilenetV2/expanded_conv_4 (--/14.02k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/depthwise (--/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_4/depthwise/depthwise_weights (3x3x192x1, 1.73k/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/expand (--/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_4/expand/weights (1x1x32x192, 6.14k/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/project (--/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_4/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_4/project/weights (1x1x192x32, 6.14k/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_5 (--/14.02k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/depthwise (--/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_5/depthwise/depthwise_weights (3x3x192x1, 1.73k/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/expand (--/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_5/expand/weights (1x1x32x192, 6.14k/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/project (--/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_5/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_5/project/weights (1x1x192x32, 6.14k/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_6 (--/20.16k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/depthwise (--/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_6/depthwise/depthwise_weights (3x3x192x1, 1.73k/1.73k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/expand (--/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_6/expand/weights (1x1x32x192, 6.14k/6.14k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/project (--/12.29k params)
FeatureExtractor/MobilenetV2/expanded_conv_6/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_6/project/weights (1x1x192x64, 12.29k/12.29k params)
FeatureExtractor/MobilenetV2/expanded_conv_7 (--/52.61k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/depthwise (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_7/depthwise/depthwise_weights (3x3x384x1, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/expand (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_7/expand/weights (1x1x64x384, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/project (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_7/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_7/project/weights (1x1x384x64, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_8 (--/52.61k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/depthwise (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_8/depthwise/depthwise_weights (3x3x384x1, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/expand (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_8/expand/weights (1x1x64x384, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/project (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_8/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_8/project/weights (1x1x384x64, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_9 (--/52.61k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/depthwise (--/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/depthwise/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_9/depthwise/depthwise_weights (3x3x384x1, 3.46k/3.46k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/expand (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/expand/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_9/expand/weights (1x1x64x384, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/project (--/24.58k params)
FeatureExtractor/MobilenetV2/expanded_conv_9/project/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/expanded_conv_9/project/weights (1x1x384x64, 24.58k/24.58k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_2_1x1_256 (--/327.68k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_2_1x1_256/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_2_1x1_256/weights (1x1x1280x256, 327.68k/327.68k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_3_1x1_128 (--/65.54k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_3_1x1_128/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_3_1x1_128/weights (1x1x512x128, 65.54k/65.54k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_4_1x1_128 (--/32.77k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_4_1x1_128/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_4_1x1_128/weights (1x1x256x128, 32.77k/32.77k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_5_1x1_64 (--/16.38k params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_5_1x1_64/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_1_Conv2d_5_1x1_64/weights (1x1x256x64, 16.38k/16.38k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_2_3x3_s2_512 (--/1.18m params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_2_3x3_s2_512/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_2_3x3_s2_512/weights (3x3x256x512, 1.18m/1.18m params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_3_3x3_s2_256 (--/294.91k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_3_3x3_s2_256/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_3_3x3_s2_256/weights (3x3x128x256, 294.91k/294.91k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_4_3x3_s2_256 (--/294.91k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_4_3x3_s2_256/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_4_3x3_s2_256/weights (3x3x128x256, 294.91k/294.91k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_5_3x3_s2_128 (--/73.73k params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_5_3x3_s2_128/BatchNorm (--/0 params)
FeatureExtractor/MobilenetV2/layer_19_2_Conv2d_5_3x3_s2_128/weights (3x3x64x128, 73.73k/73.73k params)
======================End of Report==========================
133 ops no flops stats due to incomplete shapes.
Parsing Inputs...
Incomplete shape.
=========================Options=============================
-max_depth 10000
-min_bytes 0
-min_peak_bytes 0
-min_residual_bytes 0
-min_output_bytes 0
-min_micros 0
-min_accelerator_micros 0
-min_cpu_micros 0
-min_params 0
-min_float_ops 1
-min_occurrence 0
-step -1
-order_by float_ops
-account_type_regexes .*
-start_name_regexes .*
-trim_name_regexes .*BatchNorm.*,.*Initializer.*,.*Regularizer.*,.*BiasAdd.*
-show_name_regexes .*
-hide_name_regexes
-account_displayed_op_only true
-select float_ops
-output stdout:
==================Model Analysis Report======================
Incomplete shape.
Doc:
scope: The nodes in the model graph are organized by their names, which is hierarchical like filesystem.
flops: Number of float operations. Note: Please read the implementation for the math behind it.
Profile:
node name | # float_ops
_TFProfRoot (--/13.71k flops)
MultipleGridAnchorGenerator/sub (2.17k/2.17k flops)
MultipleGridAnchorGenerator/mul_20 (2.17k/2.17k flops)
MultipleGridAnchorGenerator/mul_19 (2.17k/2.17k flops)
MultipleGridAnchorGenerator/mul_27 (1.20k/1.20k flops)
MultipleGridAnchorGenerator/mul_28 (1.20k/1.20k flops)
MultipleGridAnchorGenerator/sub_1 (1.20k/1.20k flops)
MultipleGridAnchorGenerator/mul_21 (1.08k/1.08k flops)
MultipleGridAnchorGenerator/mul_29 (600/600 flops)
MultipleGridAnchorGenerator/mul_36 (300/300 flops)
MultipleGridAnchorGenerator/mul_35 (300/300 flops)
MultipleGridAnchorGenerator/sub_2 (300/300 flops)
MultipleGridAnchorGenerator/mul_37 (150/150 flops)
MultipleGridAnchorGenerator/mul_43 (108/108 flops)
MultipleGridAnchorGenerator/mul_44 (108/108 flops)
MultipleGridAnchorGenerator/sub_3 (108/108 flops)
MultipleGridAnchorGenerator/mul_45 (54/54 flops)
MultipleGridAnchorGenerator/mul_52 (48/48 flops)
MultipleGridAnchorGenerator/mul_51 (48/48 flops)
MultipleGridAnchorGenerator/sub_4 (48/48 flops)
MultipleGridAnchorGenerator/mul_53 (24/24 flops)
MultipleGridAnchorGenerator/mul_18 (19/19 flops)
MultipleGridAnchorGenerator/mul_17 (19/19 flops)
MultipleGridAnchorGenerator/sub_5 (12/12 flops)
MultipleGridAnchorGenerator/mul_60 (12/12 flops)
MultipleGridAnchorGenerator/mul_59 (12/12 flops)
MultipleGridAnchorGenerator/mul_25 (10/10 flops)
MultipleGridAnchorGenerator/mul_26 (10/10 flops)
MultipleGridAnchorGenerator/mul_46 (6/6 flops)
MultipleGridAnchorGenerator/mul_40 (6/6 flops)
MultipleGridAnchorGenerator/mul_54 (6/6 flops)
MultipleGridAnchorGenerator/truediv_17 (6/6 flops)
MultipleGridAnchorGenerator/truediv_16 (6/6 flops)
MultipleGridAnchorGenerator/truediv_15 (6/6 flops)
MultipleGridAnchorGenerator/mul_24 (6/6 flops)
MultipleGridAnchorGenerator/mul_47 (6/6 flops)
MultipleGridAnchorGenerator/mul_48 (6/6 flops)
MultipleGridAnchorGenerator/mul_61 (6/6 flops)
MultipleGridAnchorGenerator/mul_39 (6/6 flops)
MultipleGridAnchorGenerator/mul_38 (6/6 flops)
MultipleGridAnchorGenerator/truediv_19 (6/6 flops)
MultipleGridAnchorGenerator/mul_55 (6/6 flops)
MultipleGridAnchorGenerator/mul_56 (6/6 flops)
MultipleGridAnchorGenerator/mul_32 (6/6 flops)
MultipleGridAnchorGenerator/mul_31 (6/6 flops)
MultipleGridAnchorGenerator/mul_30 (6/6 flops)
MultipleGridAnchorGenerator/truediv_18 (6/6 flops)
MultipleGridAnchorGenerator/mul_22 (6/6 flops)
MultipleGridAnchorGenerator/mul_23 (6/6 flops)
MultipleGridAnchorGenerator/mul_34 (5/5 flops)
MultipleGridAnchorGenerator/mul_33 (5/5 flops)
MultipleGridAnchorGenerator/mul_42 (3/3 flops)
MultipleGridAnchorGenerator/mul_41 (3/3 flops)
MultipleGridAnchorGenerator/mul_16 (3/3 flops)
MultipleGridAnchorGenerator/mul_15 (3/3 flops)
MultipleGridAnchorGenerator/mul_14 (3/3 flops)
MultipleGridAnchorGenerator/truediv_14 (3/3 flops)
MultipleGridAnchorGenerator/mul_50 (2/2 flops)
MultipleGridAnchorGenerator/mul_49 (2/2 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_9 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/Minimum_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_8 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_7 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_6 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_5 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_4 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_3 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_2 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/Greater (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/sub_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/sub (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/SortByField_1/Equal (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/SortByField/Equal (1/1 flops)
Preprocessor/map/while/Less_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_2 (1/1 flops)
MultipleGridAnchorGenerator/Minimum (1/1 flops)
Preprocessor/map/while/Less (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/ones/Less (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_9 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_8 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_7 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_6 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_5 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_4 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_3 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_19 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_18 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_17 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_16 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_15 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_14 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_13 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_12 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_11 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/PadOrClipBoxList/sub_10 (1/1 flops)
MultipleGridAnchorGenerator/mul_4 (1/1 flops)
MultipleGridAnchorGenerator/truediv_1 (1/1 flops)
MultipleGridAnchorGenerator/truediv (1/1 flops)
MultipleGridAnchorGenerator/mul_9 (1/1 flops)
MultipleGridAnchorGenerator/mul_8 (1/1 flops)
MultipleGridAnchorGenerator/mul_7 (1/1 flops)
MultipleGridAnchorGenerator/mul_6 (1/1 flops)
MultipleGridAnchorGenerator/mul_58 (1/1 flops)
MultipleGridAnchorGenerator/mul_57 (1/1 flops)
MultipleGridAnchorGenerator/mul_5 (1/1 flops)
MultipleGridAnchorGenerator/truediv_10 (1/1 flops)
MultipleGridAnchorGenerator/mul_3 (1/1 flops)
MultipleGridAnchorGenerator/mul_2 (1/1 flops)
MultipleGridAnchorGenerator/mul_13 (1/1 flops)
MultipleGridAnchorGenerator/mul_12 (1/1 flops)
MultipleGridAnchorGenerator/mul_11 (1/1 flops)
MultipleGridAnchorGenerator/mul_10 (1/1 flops)
MultipleGridAnchorGenerator/mul_1 (1/1 flops)
MultipleGridAnchorGenerator/mul (1/1 flops)
MultipleGridAnchorGenerator/assert_equal_1/Equal (1/1 flops)
MultipleGridAnchorGenerator/truediv_7 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/Greater (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/ChangeCoordinateFrame/truediv_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/ChangeCoordinateFrame/truediv (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/ChangeCoordinateFrame/sub_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/ChangeCoordinateFrame/sub (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/Less_1 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/Less (1/1 flops)
MultipleGridAnchorGenerator/truediv_9 (1/1 flops)
MultipleGridAnchorGenerator/truediv_8 (1/1 flops)
Postprocessor/BatchMultiClassNonMaxSuppression/map/while/MultiClassNonMaxSuppression/Minimum (1/1 flops)
MultipleGridAnchorGenerator/truediv_6 (1/1 flops)
MultipleGridAnchorGenerator/truediv_5 (1/1 flops)
MultipleGridAnchorGenerator/truediv_4 (1/1 flops)
MultipleGridAnchorGenerator/truediv_3 (1/1 flops)
MultipleGridAnchorGenerator/truediv_2 (1/1 flops)
MultipleGridAnchorGenerator/truediv_13 (1/1 flops)
MultipleGridAnchorGenerator/truediv_12 (1/1 flops)
MultipleGridAnchorGenerator/truediv_11 (1/1 flops)
======================End of Report==========================
2020-07-25 13:58:43.617738: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2020-07-25 13:58:43.638682: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.639406: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:58:43.639698: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:58:43.641314: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:58:43.652136: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:58:43.652490: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:58:43.654452: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:58:43.668766: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:58:43.682131: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:58:43.682300: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.683216: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.683942: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:58:43.690135: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz
2020-07-25 13:58:43.690364: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x1f96bc0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-07-25 13:58:43.690401: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
2020-07-25 13:58:43.747807: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.748667: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x1f96f40 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2020-07-25 13:58:43.748700: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla K80, Compute Capability 3.7
2020-07-25 13:58:43.748946: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.749642: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:58:43.749774: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:58:43.749845: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:58:43.749910: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:58:43.749992: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:58:43.750061: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:58:43.750109: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:58:43.750155: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:58:43.750290: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.751141: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.751850: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:58:43.751932: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:58:43.753643: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:58:43.753681: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:58:43.753701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:58:43.753939: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.754879: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:43.755665: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.
2020-07-25 13:58:43.755721: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-50000
I0725 13:58:43.762284 139719421781888 saver.py:1284] Restoring parameters from training/model.ckpt-50000
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/tools/freeze_graph.py:127: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
W0725 13:58:45.434061 139719421781888 deprecation.py:323] From /tensorflow-1.15.2/python3.6/tensorflow_core/python/tools/freeze_graph.py:127: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.
2020-07-25 13:58:46.126754: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:46.127580: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:58:46.127687: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:58:46.127748: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:58:46.127833: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:58:46.127901: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:58:46.127953: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:58:46.128011: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:58:46.128072: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:58:46.128226: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:46.129045: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:46.129733: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:58:46.129790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:58:46.129832: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:58:46.129859: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:58:46.130022: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:46.130933: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:46.131606: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
INFO:tensorflow:Restoring parameters from training/model.ckpt-50000
I0725 13:58:46.132998 139719421781888 saver.py:1284] Restoring parameters from training/model.ckpt-50000
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/tools/freeze_graph.py:233: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.convert_variables_to_constants`
W0725 13:58:46.763968 139719421781888 deprecation.py:323] From /tensorflow-1.15.2/python3.6/tensorflow_core/python/tools/freeze_graph.py:233: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.convert_variables_to_constants`
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.extract_sub_graph`
W0725 13:58:46.764301 139719421781888 deprecation.py:323] From /tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.extract_sub_graph`
INFO:tensorflow:Froze 324 variables.
I0725 13:58:47.168892 139719421781888 graph_util_impl.py:334] Froze 324 variables.
INFO:tensorflow:Converted 324 variables to const ops.
I0725 13:58:47.271131 139719421781888 graph_util_impl.py:394] Converted 324 variables to const ops.
2020-07-25 13:58:47.416412: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:47.417155: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Found device 0 with properties:
name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235
pciBusID: 0000:00:04.0
2020-07-25 13:58:47.417254: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-07-25 13:58:47.417312: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-07-25 13:58:47.417374: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-07-25 13:58:47.417440: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-07-25 13:58:47.417506: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-07-25 13:58:47.417572: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-07-25 13:58:47.417625: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-07-25 13:58:47.417781: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:47.418598: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:47.419368: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1767] Adding visible gpu devices: 0
2020-07-25 13:58:47.419424: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1180] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-07-25 13:58:47.419453: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1186] 0
2020-07-25 13:58:47.419471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1199] 0: N
2020-07-25 13:58:47.419724: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:47.420927: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-07-25 13:58:47.421839: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1325] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10805 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)
WARNING:tensorflow:From /content/aeroplane_detection/models/research/object_detection/exporter.py:384: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
W0725 13:58:48.085387 139719421781888 deprecation.py:323] From /content/aeroplane_detection/models/research/object_detection/exporter.py:384: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
INFO:tensorflow:No assets to save.
I0725 13:58:48.086369 139719421781888 builder_impl.py:640] No assets to save.
INFO:tensorflow:No assets to write.
I0725 13:58:48.086540 139719421781888 builder_impl.py:460] No assets to write.
INFO:tensorflow:SavedModel written to: /content/aeroplane_detection/models/research/fine_tuned_model/saved_model/saved_model.pb
I0725 13:58:48.381668 139719421781888 builder_impl.py:425] SavedModel written to: /content/aeroplane_detection/models/research/fine_tuned_model/saved_model/saved_model.pb
INFO:tensorflow:Writing pipeline config file to /content/aeroplane_detection/models/research/fine_tuned_model/pipeline.config
I0725 13:58:48.407660 139719421781888 config_util.py:254] Writing pipeline config file to /content/aeroplane_detection/models/research/fine_tuned_model/pipeline.config
In [88]:
lst = os.listdir(model_dir)
lst
Out[88]:
['model.ckpt-45640.index', 'model.ckpt-49054.data-00000-of-00001', 'model.ckpt-45640.meta', 'model.ckpt-45640.data-00000-of-00001', 'model.ckpt-46783.meta', 'model.ckpt-50000.index', 'model.ckpt-46783.data-00000-of-00001', 'model.ckpt-47922.data-00000-of-00001', 'checkpoint', 'model.ckpt-49054.meta', 'model.ckpt-50000.data-00000-of-00001', 'graph.pbtxt', 'model.ckpt-49054.index', 'model.ckpt-50000.meta', 'events.out.tfevents.1595658966.fa9e4854495f', 'model.ckpt-47922.meta', 'eval_0', 'model.ckpt-46783.index', 'export', 'model.ckpt-47922.index']
In [89]:
import os
os.path.isdir('./fine_tuned_model')
Out[89]:
True
In [90]:
!pwd
/content/aeroplane_detection/models/research
In [92]:
#downloads the frozen model that is needed for inference
files.download(output_directory + '/frozen_inference_graph.pb')
In [93]:
!zip /content/fine_tuned_model.zip /content/aeroplane_detection/models/research/fine_tuned_model
adding: content/aeroplane_detection/models/research/fine_tuned_model/ (stored 0%)
In [94]:
#downlaod the label map
files.download(DATA_BASE_PATH + '/label_map.pbtxt')
In [103]:
files.download('/content/aeroplane_detection/models/research/fine_tuned_model/model.ckpt.data-00000-of-00001')
In [95]:
def load_image_into_numpy_array(image_path):
img_data = tf.io.gfile.GFile(image_path, 'rb').read()
image = Image.open(BytesIO(img_data))
(im_width, im_height) = image.size
return np.array(image.getdata()).reshape(
(im_height, im_width, 3)).astype(np.uint8)
In [104]:
PATH_TO_TEST_IMAGES_DIR='/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane'
TEST_IMAGE_PATHS = [os.path.join(PATH_TO_TEST_IMAGES_DIR,name) for name in os.listdir(PATH_TO_TEST_IMAGES_DIR)]
IMAGE_SIZE=(12,8)
TEST_IMAGE_PATHS
Out[104]:
['/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_035.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_028.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_039.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_037.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_047.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_053.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_056.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_003.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_059.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_001.jpg', '/content/aeroplane_detection/models/research/object_detection/test_images/aeroplane/airplane_045.jpg']
In [106]:
import numpy as np
import os
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as vis_util
# path to the frozen graph:
PATH_TO_FROZEN_GRAPH = '/content/aeroplane_detection/models/research/training/export/Servo/1595685332/saved_model.pb'
# path to the label map
PATH_TO_LABEL_MAP = '/content/aeroplane_detection/data/label_map.pbtxt'
# number of classes
NUM_CLASSES = 1
#reads the frozen graph
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
label_map = label_map_util.load_labelmap(PATH_TO_LABEL_MAP)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)
# Detection
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
for image_path in TEST_IMAGE_PATHS:
image_np=load_image_into_numpy_array(image_path)
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Extract image tensor
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Extract detection boxes
boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
# Extract detection scores
scores = detection_graph.get_tensor_by_name('detection_scores:0')
# Extract detection classes
classes = detection_graph.get_tensor_by_name('detection_classes:0')
# Extract number of detections
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
# Actual detection.
(boxes, scores, classes, num_detections) = sess.run(
[boxes, scores, classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
category_index,
use_normalized_coordinates=True,
line_thickness=3,
)
plt.figure(figsize=IMAGE_SIZE)
plt.imshow(image_np)
--------------------------------------------------------------------------- DecodeError Traceback (most recent call last) <ipython-input-106-b5fb4390914f> in <module>() 19 with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid: 20 serialized_graph = fid.read() ---> 21 od_graph_def.ParseFromString(serialized_graph) 22 tf.import_graph_def(od_graph_def, name='') 23 DecodeError: Error parsing message
In [107]:
import numpy as np
import os
import six.moves.urllib as urllib
import sys
import tarfile
import tensorflow as tf
import zipfile
from collections import defaultdict
from io import StringIO
from matplotlib import pyplot as plt
from PIL import Image
from IPython.display import display
from six import BytesIO
In [108]:
from object_detection.utils import ops as utils_ops
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as vis_util
In [109]:
PATH_TO_CKPT='/content/aeroplane_detection/models/research/fine_tuned_model/frozen_inference_graph.pb'
In [110]:
detection_graph=tf.Graph()
with detection_graph.as_default():
od_graph_def=tf.GraphDef()
with tf.gfile.GFile(PATH_TO_CKPT,'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def,name='')
In [111]:
category_index = {
1: {'id': 1, 'name': 'aeroplane','display name':'aeroplane'} }
In [112]:
# final output for the prediction of the objection detection wher it could it detect the eroplane
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
for image_path in TEST_IMAGE_PATHS:
image_np=load_image_into_numpy_array(image_path)
image_np_expanded = np.expand_dims(image_np,axis=0)
image_tensor=detection_graph.get_tensor_by_name('image_tensor:0')
boxes=detection_graph.get_tensor_by_name('detection_boxes:0')
scores=detection_graph.get_tensor_by_name('detection_scores:0')
classes=detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
image_np_with_detections = image_np.copy()
(boxes,scores,classes,num_detections) = sess.run(
[boxes,scores,classes,num_detections],
feed_dict={image_tensor:image_np_expanded})
print(scores)
vis_util.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
boxes[0],
classes[0].astype(np.int32),
scores[0],
category_index,
use_normalized_coordinates=True,
min_score_thresh=.40,
agnostic_mode=False,
line_thickness=5)
plt.figure(figsize=IMAGE_SIZE)
plt.imshow(image_np_with_detections)
#print(boxes)
[[0.9880898 0.75492054 0.28821594 0.1220817 0.05134484 0.03614265 0.03549811 0.02888435 0.02793306 0.02674738 0.02612421 0.0259138 0.02165335 0.02148134 0.02138421 0.02048138]] [[0.9696908 0.12595755 0.04683277 0.0429244 0.04220244 0.03527933 0.03221607 0.02808934 0.02599037 0.02422589 0.02394769 0.02312735 0.02310377 0.02227187 0.02132019 0.02075952]] [[0.81711876 0.3357558 0.22333401 0.21135768 0.11844075 0.07662886 0.05207759 0.04864284 0.04502627 0.03707099 0.03668427 0.02940783 0.02694061 0.02586237 0.02422369 0.02319145]] [[0.9352028 0.06365266 0.0281589 0.01865971 0.01846799 0.01819268 0.01757702 0.01706168 0.01599604 0.01597474 0.01595688 0.01547667 0.01518637 0.01502615 0.01480466 0.01478502]] [[0.9811863 0.30832797 0.06693786 0.04480621 0.03977346 0.03943837 0.03843531 0.0367327 0.0343627 0.03207234 0.03051218 0.02756101 0.02659786 0.02612856 0.02559778 0.02508923]] [[0.99450445 0.96765393 0.96062315 0.7957505 0.07672852 0.04216003 0.0398443 0.03447589 0.02336958 0.02274361 0.02020833 0.01754251 0.01713607 0.01638028 0.01598057 0.01492342]] [[0.99755454 0.9967127 0.98925495 0.97751164 0.6247861 0.04804254 0.02985069 0.02952871 0.0270676 0.02686656 0.02486342 0.02215517 0.0220347 0.02160177 0.02150038 0.02137378]] [[0.96036875 0.03305274 0.03228962 0.02774081 0.02389216 0.02201214 0.01985681 0.01959479 0.01958141 0.01820162 0.01744121 0.01729932 0.01690543 0.01686248 0.01489347 0.01473629]] [[0.9848331 0.16941205 0.09566492 0.02573195 0.02487653 0.02127132 0.01885572 0.01874828 0.01863116 0.01801518 0.01773006 0.01685578 0.01653004 0.01649293 0.0156458 0.01544511]] [[0.995059 0.9496852 0.9454798 0.2895307 0.06289065 0.02107072 0.01799351 0.01766181 0.01747555 0.01693135 0.01616612 0.01533136 0.01526466 0.01478341 0.01452124 0.01426035]] [[0.9896114 0.9872904 0.964869 0.62034464 0.46692285 0.23219839 0.16989237 0.05915147 0.05760747 0.05299243 0.05218735 0.04877356 0.04516762 0.03436396 0.0342963 0.033838 ]]
In [ ]: